2015-10-21T06:10:01.883Z I CONTROL [signalProcessingThread] pid=16473 port=27017 64-bit host=ip-172-19-100-23 2015-10-21T06:10:01.883Z I CONTROL [signalProcessingThread] db version v3.0.5 2015-10-21T06:10:01.883Z I CONTROL [signalProcessingThread] git version: 8bc4ae20708dbb493cb09338d9e7be6698e4a3a3 2015-10-21T06:10:01.883Z I CONTROL [signalProcessingThread] build info: Linux ip-10-230-137-3 3.4.43-43.43.amzn1.x86_64 #1 SMP Mon May 6 18:04:41 UTC 2013 x86_64 BOOST_LIB_VERSION=1_49 2015-10-21T06:10:01.883Z I CONTROL [signalProcessingThread] allocator: tcmalloc 2015-10-21T06:10:01.883Z I CONTROL [signalProcessingThread] options: { config: "/etc/mongod.conf", net: { port: 27017, ssl: { PEMKeyFile: "/mongodata/config/mongodb.pem", PEMKeyPassword: "", mode: "preferSSL" } }, operationProfiling: { mode: "off", slowOpThresholdMs: 100 }, processManagement: { fork: true, pidFilePath: "/var/run/mongodb/mongod.pid" }, replication: { oplogSizeMB: 20480, replSetName: "upe_dccb" }, security: { clusterAuthMode: "keyFile", keyFile: "/mongodata/config/keyfile" }, storage: { dbPath: "/mongodata/db", directoryPerDB: true, engine: "wiredTiger", wiredTiger: { collectionConfig: { blockCompressor: "snappy" }, engineConfig: { directoryForIndexes: true, journalCompressor: "snappy", statisticsLogDelaySecs: 0 }, indexConfig: { prefixCompression: true } } }, systemLog: { destination: "file", logAppend: true, logRotate: "reopen", path: "/data/log/mongodb/mongodb.log", timeStampFormat: "iso8601-utc", traceAllExceptions: false, verbosity: 0 } } 2015-10-21T21:29:41.374Z I REPL [conn169748] replSetReconfig admin command received from client 2015-10-21T21:29:41.457Z I REPL [conn169748] replSetReconfig config object with 3 members parses ok 2015-10-21T21:29:41.458Z I REPL [ReplicationExecutor] New replica set config in use: { _id: "upe_dccb", version: 11496652, members: [ { _id: 4, host: "mongodb-upe_dccb-1.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017", arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 5, host: "mongodb-upe_dccb-2.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017", arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 6, host: "mongodb-upe_dccb-3.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017", arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 } ], settings: { chainingAllowed: true, heartbeatTimeoutSecs: 10, getLastErrorModes: {}, getLastErrorDefaults: { w: 1, wtimeout: 0 } } } 2015-10-21T21:29:41.458Z I REPL [ReplicationExecutor] This node is mongodb-upe_dccb-1.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 in the config 2015-10-21T22:11:39.385Z I INDEX [conn170327] build index on: sledge.scheduled_messages properties: { v: 1, key: { executed_at: 1, scheduled_at: 1 }, name: "executed_at_1_scheduled_at_1", ns: "sledge.scheduled_messages", background: true } 2015-10-21T22:11:39.385Z I INDEX [conn170327] build index done. scanned 0 total records. 0 secs 2015-10-21T22:11:39.391Z I INDEX [conn170327] build index on: sledge.scheduled_messages properties: { v: 1, key: { scheduled_at: 1, executed_at: 1 }, name: "scheduled_at_1_executed_at_1", ns: "sledge.scheduled_messages", background: true } 2015-10-21T22:11:39.391Z I INDEX [conn170327] build index done. scanned 0 total records. 0 secs 2015-10-21T22:11:39.398Z I INDEX [conn170328] build index on: sledge.message_schedules properties: { v: 1, key: { scheduled_at: 1 }, name: "scheduled_at_1", ns: "sledge.message_schedules", background: true } 2015-10-21T22:11:39.398Z I INDEX [conn170328] build index done. scanned 0 total records. 0 secs 2015-10-21T22:11:44.674Z I INDEX [conn170330] build index on: configuration.config_entries properties: { v: 1, key: { context: 1 }, name: "context_1", ns: "configuration.config_entries", background: true } 2015-10-21T22:11:44.675Z I INDEX [conn170330] build index done. scanned 137 total records. 0 secs 2015-10-21T22:11:44.684Z I INDEX [conn170330] build index on: configuration.config_entries properties: { v: 1, key: { context: 1, key: 1 }, name: "context_1_key_1", ns: "configuration.config_entries", background: true } 2015-10-21T22:11:44.686Z I INDEX [conn170330] build index done. scanned 137 total records. 0 secs 2015-10-21T22:12:05.079Z I INDEX [conn170338] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, info.isSecurityVm: 1 }, name: "customer_id_1_info.isSecurityVm_1", ns: "sledge.endpoints", background: true } 2015-10-21T22:12:07.960Z I INDEX [conn170338] build index done. scanned 231932 total records. 2 secs 2015-10-21T22:12:09.069Z I INDEX [conn170338] build index on: sledge.endpoint_devices properties: { v: 1, key: { customer_id: 1, detected_at: 1 }, name: "customer_id_1_detected_at_1", ns: "sledge.endpoint_devices", background: true } 2015-10-21T22:12:09.070Z I INDEX [conn170338] build index done. scanned 100 total records. 0 secs 2015-10-21T22:12:09.075Z I INDEX [conn170338] build index on: sledge.endpoint_devices properties: { v: 1, key: { endpoint_id: 1, device_id: 1, instance_id: 1 }, name: "endpoint_id_1_device_id_1_instance_id_1", ns: "sledge.endpoint_devices", background: true } 2015-10-21T22:12:09.077Z I INDEX [conn170338] build index done. scanned 100 total records. 0 secs 2015-10-21T22:12:09.082Z I INDEX [conn170338] build index on: sledge.endpoint_devices properties: { v: 1, key: { endpoint_id: 1, device_id: 1, device_type: 1, instance_id: 1, description: 1, detected_at: 1 }, name: "endpoint_id_1_device_id_1_device_type_1_instance_id_1_description_1_detected_at_1", ns: "sledge.endpoint_devices", background: true } 2015-10-21T22:12:09.084Z I INDEX [conn170338] build index done. scanned 100 total records. 0 secs 2015-10-21T22:12:09.089Z I INDEX [conn170338] build index on: sledge.endpoint_devices properties: { v: 1, key: { customer_id: 1, device_type: 1, detected_at: 1 }, name: "customer_id_1_device_type_1_detected_at_1", ns: "sledge.endpoint_devices", background: true } 2015-10-21T22:12:09.091Z I INDEX [conn170338] build index done. scanned 100 total records. 0 secs 2015-10-21T22:12:09.186Z I INDEX [conn170338] build index on: sledge.endpoint_threats properties: { v: 1, key: { endpoint_id: 1, threat_id: 1 }, name: "endpoint_id_1_threat_id_1", ns: "sledge.endpoint_threats", background: true } 2015-10-21T22:12:09.187Z I INDEX [conn170338] build index done. scanned 66 total records. 0 secs 2015-10-21T22:12:09.192Z I INDEX [conn170338] build index on: sledge.endpoint_threats properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.endpoint_threats", background: true } 2015-10-21T22:12:09.193Z I INDEX [conn170338] build index done. scanned 66 total records. 0 secs 2015-10-21T22:15:23.360Z I INDEX [conn170328] build index on: sledge.mcs_commands properties: { v: 1, key: { endpoint_id: 1, command_id: 1 }, name: "endpoint_id_1_command_id_1", ns: "sledge.mcs_commands", background: true } 2015-10-21T22:15:23.361Z I INDEX [conn170328] build index done. scanned 0 total records. 0 secs 2015-10-21T22:15:23.364Z I INDEX [conn170328] build index on: sledge.mcs_commands properties: { v: 1, key: { endpoint_id: 1, command_app_id: 1 }, name: "endpoint_id_1_command_app_id_1", ns: "sledge.mcs_commands", background: true } 2015-10-21T22:15:23.365Z I INDEX [conn170328] build index done. scanned 0 total records. 0 secs 2015-10-21T22:15:23.369Z I INDEX [conn170328] build index on: sledge.mcs_commands properties: { v: 1, key: { endpoint_id: 1, posted_app_id: 1, body_sha: 1 }, name: "endpoint_id_1_posted_app_id_1_body_sha_1", ns: "sledge.mcs_commands", background: true } 2015-10-21T22:15:23.369Z I INDEX [conn170328] build index done. scanned 0 total records. 0 secs 2015-10-21T22:15:23.373Z I INDEX [conn170328] build index on: sledge.mcs_commands properties: { v: 1, key: { created_at: 1 }, name: "created_at_1", ns: "sledge.mcs_commands", background: true } 2015-10-21T22:15:23.373Z I INDEX [conn170328] build index done. scanned 0 total records. 0 secs 2015-10-21T22:59:53.788Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-21T23:15:34.390Z I INDEX [conn170471] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.report_bases", background: true } 2015-10-21T23:15:34.393Z I INDEX [conn170471] build index done. scanned 247 total records. 0 secs 2015-10-21T23:15:34.397Z I INDEX [conn170471] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1 }, name: "customer_id_1__type_1", ns: "sledge.report_bases", background: true } 2015-10-21T23:15:34.400Z I INDEX [conn170471] build index done. scanned 247 total records. 0 secs 2015-10-21T23:15:34.405Z I INDEX [conn170471] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, tm: 1 }, name: "customer_id_1__type_1_tm_1", ns: "sledge.report_bases", background: true } 2015-10-21T23:15:34.408Z I INDEX [conn170471] build index done. scanned 247 total records. 0 secs 2015-10-21T23:15:34.412Z I INDEX [conn170471] build index on: sledge.report_bases properties: { v: 1, key: { ham_idx_upd: -1 }, name: "ham_idx_upd_-1", ns: "sledge.report_bases", background: true, sparse: true } 2015-10-21T23:15:34.413Z I INDEX [conn170471] build index done. scanned 247 total records. 0 secs 2015-10-21T23:15:34.417Z I INDEX [conn170471] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, name: 1, group: 1 }, name: "customer_id_1_name_1_group_1", ns: "sledge.report_bases", background: true } 2015-10-21T23:15:34.420Z I INDEX [conn170471] build index done. scanned 247 total records. 0 secs 2015-10-21T23:15:34.425Z I INDEX [conn170471] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, hits.type: 1, name: 1, group: 1 }, name: "customer_id_1_hits.type_1_name_1_group_1", ns: "sledge.report_bases", background: true } 2015-10-21T23:15:34.428Z I INDEX [conn170471] build index done. scanned 247 total records. 0 secs 2015-10-21T23:15:34.433Z I INDEX [conn170471] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, name: 1 }, name: "customer_id_1__type_1_name_1", ns: "sledge.report_bases", background: true } 2015-10-21T23:15:34.436Z I INDEX [conn170471] build index done. scanned 247 total records. 0 secs 2015-10-21T23:15:41.920Z I INDEX [conn170471] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, hidden: 1, alert_type: 1 }, name: "customer_id_1_hidden_1_alert_type_1", ns: "sledge.alerts", background: true } 2015-10-21T23:15:41.920Z I INDEX [conn170471] build index done. scanned 0 total records. 0 secs 2015-10-21T23:15:41.924Z I INDEX [conn170471] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, unhide_at: 1 }, name: "customer_id_1_unhide_at_1", ns: "sledge.alerts", background: true } 2015-10-21T23:15:41.924Z I INDEX [conn170471] build index done. scanned 0 total records. 0 secs 2015-10-21T23:15:41.929Z I INDEX [conn170471] build index on: sledge.alerts properties: { v: 1, key: { unhide_at: 1, hidden: 1 }, name: "unhide_at_1_hidden_1", ns: "sledge.alerts", background: true } 2015-10-21T23:15:41.929Z I INDEX [conn170471] build index done. scanned 0 total records. 0 secs 2015-10-21T23:15:41.933Z I INDEX [conn170471] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, data.endpoint_id: 1, alert_type: 1 }, name: "customer_id_1_data.endpoint_id_1_alert_type_1", ns: "sledge.alerts", background: true } 2015-10-21T23:15:41.933Z I INDEX [conn170471] build index done. scanned 0 total records. 0 secs 2015-10-21T23:15:41.937Z I INDEX [conn170471] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, data.threat_id: 1, alert_type: 1 }, name: "customer_id_1_data.threat_id_1_alert_type_1", ns: "sledge.alerts", background: true } 2015-10-21T23:15:41.938Z I INDEX [conn170471] build index done. scanned 0 total records. 0 secs 2015-10-21T23:15:41.942Z I INDEX [conn170471] build index on: sledge.alerts properties: { v: 1, key: { endpoint_id: 1 }, name: "endpoint_id_1", ns: "sledge.alerts", background: true, sparse: true } 2015-10-21T23:15:41.942Z I INDEX [conn170471] build index done. scanned 0 total records. 0 secs 2015-10-21T23:16:04.797Z I INDEX [conn170426] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, dom: 1 }, name: "customer_id_1_dom_1", ns: "sledge.report_bases", background: true } 2015-10-21T23:16:04.800Z I INDEX [conn170426] build index done. scanned 249 total records. 0 secs 2015-10-21T23:16:04.806Z I INDEX [conn170426] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, cat: 1 }, name: "customer_id_1_cat_1", ns: "sledge.report_bases", background: true } 2015-10-21T23:16:04.809Z I INDEX [conn170426] build index done. scanned 249 total records. 0 secs 2015-10-21T23:16:04.814Z I INDEX [conn170426] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, rsn: 1 }, name: "customer_id_1_rsn_1", ns: "sledge.report_bases", background: true } 2015-10-21T23:16:04.817Z I INDEX [conn170426] build index done. scanned 249 total records. 0 secs 2015-10-21T23:16:04.823Z I INDEX [conn170426] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, hits.type: 1, dom: 1 }, name: "customer_id_1_hits.type_1_dom_1", ns: "sledge.report_bases", background: true } 2015-10-21T23:16:04.826Z I INDEX [conn170426] build index done. scanned 249 total records. 0 secs 2015-10-21T23:16:04.832Z I INDEX [conn170426] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, dom: 1, cat: 1, rsn: 1 }, name: "customer_id_1__type_1_dom_1_cat_1_rsn_1", ns: "sledge.report_bases", background: true } 2015-10-21T23:16:04.835Z I INDEX [conn170426] build index done. scanned 249 total records. 0 secs 2015-10-21T23:16:04.866Z I INDEX [conn170426] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, usr: 1 }, name: "customer_id_1_usr_1", ns: "sledge.report_bases", background: true } 2015-10-21T23:16:04.869Z I INDEX [conn170426] build index done. scanned 250 total records. 0 secs 2015-10-21T23:16:04.875Z I INDEX [conn170426] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, loc: 1 }, name: "customer_id_1_loc_1", ns: "sledge.report_bases", background: true } 2015-10-21T23:16:04.878Z I INDEX [conn170426] build index done. scanned 250 total records. 0 secs 2015-10-21T23:16:04.883Z I INDEX [conn170426] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, hits.type: 1, usr: 1 }, name: "customer_id_1_hits.type_1_usr_1", ns: "sledge.report_bases", background: true } 2015-10-21T23:16:04.888Z I INDEX [conn170426] build index done. scanned 250 total records. 0 secs 2015-10-21T23:16:04.893Z I INDEX [conn170426] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, usr: 1, loc: 1 }, name: "customer_id_1__type_1_usr_1_loc_1", ns: "sledge.report_bases", background: true } 2015-10-21T23:16:04.897Z I INDEX [conn170426] build index done. scanned 250 total records. 0 secs 2015-10-21T23:59:53.659Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T00:59:53.509Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T01:59:53.506Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T02:59:53.424Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T03:59:53.454Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T04:59:53.689Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T05:44:53.116Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T05:59:54.894Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T06:59:54.332Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T07:59:53.509Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T08:59:54.771Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T09:59:53.253Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T10:59:54.678Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T11:59:54.404Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T12:59:53.334Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T13:59:53.463Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T14:59:53.242Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T15:59:53.186Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T16:59:53.012Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T17:59:53.498Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T18:59:53.011Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T19:59:54.205Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T20:59:54.293Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T21:59:54.470Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T22:59:54.615Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-22T23:59:54.568Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T00:59:54.074Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T01:59:55.276Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T02:59:54.573Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T03:59:55.280Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T04:59:55.680Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T05:44:52.415Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T05:59:54.241Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T06:59:54.043Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T07:59:54.468Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T08:59:53.644Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T09:59:54.056Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T10:59:54.474Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T11:59:54.214Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T12:59:53.856Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T13:59:53.375Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T14:59:53.808Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T15:59:53.451Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T16:59:53.710Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T18:00:57.111Z I INDEX [conn183772] build index on: sledge.rendered_policies properties: { v: 1, key: { policy_fragment_ids: 1 }, name: "policy_fragment_ids_1", ns: "sledge.rendered_policies", sparse: false, background: true } 2015-10-23T18:00:57.111Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:01:51.916Z I INDEX [conn183772] build index on: sledge.customers properties: { v: 1, key: { licenses.expires_on: 1 }, name: "licenses.expires_on_1", ns: "sledge.customers", sparse: false, background: true } 2015-10-23T18:01:51.916Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:02:19.141Z I INDEX [conn183772] build index on: sledge.customers properties: { v: 1, key: { beta: 1 }, name: "beta_1", ns: "sledge.customers", sparse: true, background: true } 2015-10-23T18:02:20.725Z I INDEX [conn183772] build index done. scanned 0 total records. 1 secs 2015-10-23T18:02:25.240Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T18:03:53.398Z I INDEX [conn183772] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1 }, name: "c_1__type_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-23T18:03:53.398Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:04:01.580Z I INDEX [conn183772] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, name_lowercase: 1 }, name: "c_1__type_1_name_lowercase_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-23T18:04:01.580Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:04:19.768Z I INDEX [conn183772] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, search_tokens: 1 }, name: "c_1__type_1_search_tokens_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-23T18:04:19.768Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:04:31.425Z I INDEX [conn183772] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, external_id: 1 }, name: "c_1__type_1_external_id_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-23T18:04:31.425Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:04:43.227Z I INDEX [conn183772] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, origin: 1 }, name: "c_1__type_1_origin_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-23T18:04:43.228Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:04:46.467Z I INDEX [conn183772] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, email: 1 }, name: "c_1__type_1_email_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-23T18:04:46.467Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:05:11.096Z I INDEX [conn183772] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, dn: 1 }, name: "c_1__type_1_dn_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-23T18:05:11.310Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:06:20.928Z I INDEX [conn183772] build index on: sledge.policies properties: { v: 1, key: { customer_id: 1, target: 1, priority: -1 }, name: "customer_id_1_target_1_priority_-1", ns: "sledge.policies", sparse: false, background: true } 2015-10-23T18:06:20.928Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:06:33.049Z I INDEX [conn183772] build index on: sledge.policies properties: { v: 1, key: { enabled: 1, disable_at: 1 }, name: "enabled_1_disable_at_1", ns: "sledge.policies", sparse: false, background: true } 2015-10-23T18:06:33.049Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:10:47.541Z I INDEX [conn183772] build index on: sledge.mcs_policy_bases properties: { v: 1, key: { customer_id: 1, _type: 1 }, name: "customer_id_1__type_1", ns: "sledge.mcs_policy_bases", sparse: false, background: true } 2015-10-23T18:10:47.541Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:11:30.259Z I INDEX [conn183772] build index on: sledge.global_exclusions properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.global_exclusions", sparse: false, background: true } 2015-10-23T18:11:30.260Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:11:42.539Z I INDEX [conn183772] build index on: sledge.customer_flags properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.customer_flags", sparse: false, background: true } 2015-10-23T18:11:42.539Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:19:12.818Z I INDEX [conn183772] build index on: sledge.directory_user_matches properties: { v: 1, key: { local_endpoint_id: 1 }, name: "local_endpoint_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T18:19:12.818Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:19:34.626Z I INDEX [conn183772] build index on: sledge.directory_user_matches properties: { v: 1, key: { name_lowercase: 1 }, name: "name_lowercase_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T18:19:34.626Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:19:46.815Z I INDEX [conn183772] build index on: sledge.directory_user_matches properties: { v: 1, unique: true, key: { name_lowercase: 1, c: 1 }, name: "name_lowercase_1_c_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T18:19:46.816Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:20:08.418Z I INDEX [conn183772] build index on: sledge.directory_user_matches properties: { v: 1, key: { user_id: 1 }, name: "user_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T18:20:08.418Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:20:11.544Z I INDEX [conn183772] build index on: sledge.directory_user_matches properties: { v: 1, key: { customer_id: 1, name_lowercase: 1 }, name: "customer_id_1_name_lowercase_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T18:20:11.746Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:20:14.495Z I INDEX [conn183772] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, local_endpoint_id: 1 }, name: "c_1_local_endpoint_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T18:20:14.496Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:20:21.854Z I INDEX [conn183772] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, name_lowercase: 1 }, name: "c_1_name_lowercase_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T18:20:21.855Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:20:44.106Z I INDEX [conn183772] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, external_id: 1 }, name: "c_1_external_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T18:20:44.106Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:21:12.849Z I INDEX [conn183772] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, origin: 1 }, name: "c_1_origin_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T18:21:12.850Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:21:15.782Z I INDEX [conn183772] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, name: 1 }, name: "c_1_name_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T18:21:15.783Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:21:40.813Z I INDEX [conn183772] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, dn: 1 }, name: "c_1_dn_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T18:21:40.813Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:22:01.185Z I INDEX [conn183772] build index on: sledge.endpoints properties: { v: 1, key: { user_match_id: 1 }, name: "user_match_id_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T18:22:01.185Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:22:04.435Z I INDEX [conn183772] build index on: sledge.endpoints properties: { v: 1, key: { domain_name: 1, computer_name: 1, operating_system: 1 }, name: "domain_name_1_computer_name_1_operating_system_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T18:22:04.435Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:22:39.794Z I INDEX [conn183772] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T18:22:39.794Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:22:46.464Z I INDEX [conn183772] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, feature_codes: 1 }, name: "customer_id_1_feature_codes_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T18:22:46.464Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:22:52.645Z I INDEX [conn183772] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, label: 1 }, name: "customer_id_1_label_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T18:22:52.645Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:23:01.411Z I INDEX [conn183772] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, label_lowercase: 1 }, name: "customer_id_1_label_lowercase_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T18:23:01.411Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:23:14.709Z I INDEX [conn183772] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, user_match_id: 1 }, name: "customer_id_1_user_match_id_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T18:23:14.710Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:23:19.211Z I INDEX [conn183772] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, search_tokens: 1 }, name: "customer_id_1_search_tokens_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T18:23:19.211Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:23:34.212Z I INDEX [conn183772] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, transport: 1 }, name: "customer_id_1_transport_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T18:23:34.212Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:23:43.310Z I INDEX [conn183772] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, endpoint_type: 1 }, name: "customer_id_1_endpoint_type_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T18:23:46.017Z I INDEX [conn183772] build index done. scanned 0 total records. 2 secs 2015-10-23T18:24:48.803Z I INDEX [conn183772] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, info.platform: 1 }, name: "customer_id_1_info.platform_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T18:24:48.803Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:24:56.537Z I INDEX [conn183772] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, registered_at: 1 }, name: "customer_id_1_registered_at_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T18:24:59.546Z I INDEX [conn183772] build index done. scanned 0 total records. 3 secs 2015-10-23T18:25:19.230Z I INDEX [conn183772] build index on: sledge.endpoints properties: { v: 1, key: { last_connected_from: 1 }, name: "last_connected_from_1", ns: "sledge.endpoints", sparse: true, background: true } 2015-10-23T18:25:19.230Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:25:22.155Z I INDEX [conn183772] build index on: sledge.endpoints properties: { v: 1, key: { beta: 1 }, name: "beta_1", ns: "sledge.endpoints", sparse: true, background: true } 2015-10-23T18:25:22.155Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:26:14.228Z I INDEX [conn183772] build index on: sledge.policy_assignments properties: { v: 1, key: { endpoint_id: 1, policy_type: 1 }, name: "endpoint_id_1_policy_type_1", ns: "sledge.policy_assignments", sparse: false, background: true } 2015-10-23T18:26:14.228Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:26:32.897Z I INDEX [conn183772] build index on: sledge.policy_assignments properties: { v: 1, key: { endpoint_id: 1 }, name: "endpoint_id_1", ns: "sledge.policy_assignments", sparse: false, background: true } 2015-10-23T18:26:35.614Z I INDEX [conn183772] build index done. scanned 0 total records. 2 secs 2015-10-23T18:28:01.499Z I INDEX [conn183772] build index on: sledge.event_bases properties: { v: 1, key: { customer_id: 1, severity: 1, reported_at: 1, _type: 1 }, name: "customer_id_1_severity_1_reported_at_1__type_1", ns: "sledge.event_bases", sparse: false, background: true } 2015-10-23T18:28:01.499Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:28:05.205Z I INDEX [conn183772] build index on: sledge.event_bases properties: { v: 1, key: { customer_id: 1, reported_at: -1, _type: 1 }, name: "customer_id_1_reported_at_-1__type_1", ns: "sledge.event_bases", sparse: false, background: true } 2015-10-23T18:28:05.398Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:28:11.113Z I INDEX [conn183772] build index on: sledge.event_bases properties: { v: 1, key: { _type: 1, reported_at: 1 }, name: "_type_1_reported_at_1", ns: "sledge.event_bases", sparse: false, background: true } 2015-10-23T18:28:11.113Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:28:28.739Z I INDEX [conn183772] build index on: sledge.event_bases properties: { v: 1, key: { reported_at: 1 }, name: "reported_at_1", ns: "sledge.event_bases", sparse: false, background: true } 2015-10-23T18:28:28.739Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:28:28.948Z I INDEX [conn183772] build index on: sledge.event_bases properties: { v: 1, key: { customer_id: 1, inserted_at: 1 }, name: "customer_id_1_inserted_at_1", ns: "sledge.event_bases", sparse: true, background: true } 2015-10-23T18:28:28.948Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:28:32.433Z I INDEX [conn183772] build index on: sledge.event_bases properties: { v: 1, key: { endpoint_id: 1, _type: 1, reported_at: 1 }, name: "endpoint_id_1__type_1_reported_at_1", ns: "sledge.event_bases", sparse: false, background: true } 2015-10-23T18:28:32.433Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T18:28:51.057Z I INDEX [conn183772] build index on: sledge.event_bases properties: { v: 1, key: { user_match_id: 1 }, name: "user_match_id_1", ns: "sledge.event_bases", sparse: true, background: true } 2015-10-23T18:28:51.057Z I INDEX [conn183772] build index done. scanned 0 total records. 0 secs 2015-10-23T19:21:11.917Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T20:04:43.489Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T20:38:38.015Z I NETWORK [conn184862] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:43362] 2015-10-23T20:38:43.759Z I NETWORK [conn184855] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:43484] 2015-10-23T20:44:45.582Z I NETWORK [conn184782] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:43356] 2015-10-23T21:03:40.745Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T21:05:36.776Z I INDEX [conn185019] build index on: scalability.MetaEndpoints properties: { v: 1, key: { cust_id: 1.0 }, name: "cust_id_1", ns: "scalability.MetaEndpoints" } 2015-10-23T21:05:36.776Z I INDEX [conn185019] building index using bulk method 2015-10-23T21:06:29.957Z I - [conn185019] Index: (2/3) BTree Bottom Up Progress: 100/500 20% 2015-10-23T21:06:29.957Z I INDEX [conn185019] done building bottom layer, going to commit 2015-10-23T21:06:29.961Z I INDEX [conn185019] build index done. scanned 500 total records. 53 secs 2015-10-23T22:02:20.510Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T22:04:10.461Z I INDEX [conn185362] build index on: sledge.rendered_policies properties: { v: 1, key: { policy_fragment_ids: 1 }, name: "policy_fragment_ids_1", ns: "sledge.rendered_policies", sparse: false, background: true } 2015-10-23T22:04:10.461Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:04:34.716Z I INDEX [conn185362] build index on: sledge.customers properties: { v: 1, key: { licenses.expires_on: 1 }, name: "licenses.expires_on_1", ns: "sledge.customers", sparse: false, background: true } 2015-10-23T22:04:34.716Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:04:40.025Z I INDEX [conn185362] build index on: sledge.customers properties: { v: 1, key: { beta: 1 }, name: "beta_1", ns: "sledge.customers", sparse: true, background: true } 2015-10-23T22:04:41.843Z I INDEX [conn185362] build index done. scanned 0 total records. 1 secs 2015-10-23T22:05:01.975Z I INDEX [conn185362] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1 }, name: "c_1__type_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-23T22:05:01.975Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:05:14.575Z I INDEX [conn185362] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, name_lowercase: 1 }, name: "c_1__type_1_name_lowercase_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-23T22:05:14.749Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:05:17.390Z I INDEX [conn185362] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, search_tokens: 1 }, name: "c_1__type_1_search_tokens_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-23T22:05:17.391Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:05:23.724Z I INDEX [conn185362] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, external_id: 1 }, name: "c_1__type_1_external_id_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-23T22:05:23.724Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:05:30.040Z I INDEX [conn185362] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, origin: 1 }, name: "c_1__type_1_origin_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-23T22:05:30.040Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:05:42.148Z I INDEX [conn185362] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, email: 1 }, name: "c_1__type_1_email_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-23T22:05:42.148Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:05:44.578Z I INDEX [conn185362] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, dn: 1 }, name: "c_1__type_1_dn_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-23T22:05:44.578Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:07:41.352Z I INDEX [conn185362] build index on: sledge.policies properties: { v: 1, key: { customer_id: 1, target: 1, priority: -1 }, name: "customer_id_1_target_1_priority_-1", ns: "sledge.policies", sparse: false, background: true } 2015-10-23T22:07:41.352Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:07:48.669Z I INDEX [conn185362] build index on: sledge.policies properties: { v: 1, key: { enabled: 1, disable_at: 1 }, name: "enabled_1_disable_at_1", ns: "sledge.policies", sparse: false, background: true } 2015-10-23T22:07:48.669Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:09:16.346Z I INDEX [conn185362] build index on: sledge.mcs_policy_bases properties: { v: 1, key: { customer_id: 1, _type: 1 }, name: "customer_id_1__type_1", ns: "sledge.mcs_policy_bases", sparse: false, background: true } 2015-10-23T22:09:16.346Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:10:01.702Z I INDEX [conn185362] build index on: sledge.global_exclusions properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.global_exclusions", sparse: false, background: true } 2015-10-23T22:10:01.702Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:10:11.813Z I INDEX [conn185362] build index on: sledge.customer_flags properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.customer_flags", sparse: false, background: true } 2015-10-23T22:10:11.813Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:12:11.047Z I INDEX [conn185362] build index on: sledge.directory_user_matches properties: { v: 1, key: { local_endpoint_id: 1 }, name: "local_endpoint_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T22:12:11.047Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:12:29.731Z I INDEX [conn185362] build index on: sledge.directory_user_matches properties: { v: 1, key: { name_lowercase: 1 }, name: "name_lowercase_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T22:12:29.731Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:12:32.477Z I INDEX [conn185362] build index on: sledge.directory_user_matches properties: { v: 1, unique: true, key: { name_lowercase: 1, c: 1 }, name: "name_lowercase_1_c_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T22:12:32.478Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:12:39.928Z I INDEX [conn185362] build index on: sledge.directory_user_matches properties: { v: 1, key: { user_id: 1 }, name: "user_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T22:12:39.928Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:12:53.950Z I INDEX [conn185362] build index on: sledge.directory_user_matches properties: { v: 1, key: { customer_id: 1, name_lowercase: 1 }, name: "customer_id_1_name_lowercase_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T22:12:53.951Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:12:59.150Z I INDEX [conn185362] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, local_endpoint_id: 1 }, name: "c_1_local_endpoint_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T22:12:59.150Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:13:01.903Z I INDEX [conn185362] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, name_lowercase: 1 }, name: "c_1_name_lowercase_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T22:13:01.903Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:13:22.377Z I INDEX [conn185362] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, external_id: 1 }, name: "c_1_external_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T22:13:22.377Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:13:27.387Z I INDEX [conn185362] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, origin: 1 }, name: "c_1_origin_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T22:13:27.387Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:13:42.704Z I INDEX [conn185362] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, name: 1 }, name: "c_1_name_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T22:13:42.705Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:13:45.099Z I INDEX [conn185362] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, dn: 1 }, name: "c_1_dn_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-23T22:13:45.099Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:14:13.531Z I INDEX [conn185362] build index on: sledge.endpoints properties: { v: 1, key: { user_match_id: 1 }, name: "user_match_id_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T22:14:13.531Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:14:15.932Z I INDEX [conn185362] build index on: sledge.endpoints properties: { v: 1, key: { domain_name: 1, computer_name: 1, operating_system: 1 }, name: "domain_name_1_computer_name_1_operating_system_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T22:14:16.426Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:14:22.374Z I INDEX [conn185362] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T22:14:25.143Z I INDEX [conn185362] build index done. scanned 0 total records. 2 secs 2015-10-23T22:14:29.996Z I INDEX [conn185362] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, feature_codes: 1 }, name: "customer_id_1_feature_codes_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T22:14:29.996Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:14:34.508Z I INDEX [conn185362] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, label: 1 }, name: "customer_id_1_label_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T22:14:34.508Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:14:43.238Z I INDEX [conn185362] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, label_lowercase: 1 }, name: "customer_id_1_label_lowercase_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T22:14:43.239Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:14:45.651Z I INDEX [conn185362] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, user_match_id: 1 }, name: "customer_id_1_user_match_id_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T22:14:45.651Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:14:54.571Z I INDEX [conn185362] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, search_tokens: 1 }, name: "customer_id_1_search_tokens_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T22:14:54.571Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:15:01.887Z I INDEX [conn185362] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, transport: 1 }, name: "customer_id_1_transport_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T22:15:01.887Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:15:04.997Z I INDEX [conn185362] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, endpoint_type: 1 }, name: "customer_id_1_endpoint_type_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T22:15:04.997Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:15:07.916Z I INDEX [conn185362] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, info.platform: 1 }, name: "customer_id_1_info.platform_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T22:15:07.916Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:15:10.327Z I INDEX [conn185362] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, registered_at: 1 }, name: "customer_id_1_registered_at_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-23T22:15:10.327Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:15:31.593Z I INDEX [conn185362] build index on: sledge.endpoints properties: { v: 1, key: { last_connected_from: 1 }, name: "last_connected_from_1", ns: "sledge.endpoints", sparse: true, background: true } 2015-10-23T22:15:31.761Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:15:41.667Z I INDEX [conn185362] build index on: sledge.endpoints properties: { v: 1, key: { beta: 1 }, name: "beta_1", ns: "sledge.endpoints", sparse: true, background: true } 2015-10-23T22:15:41.668Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:17:01.566Z I INDEX [conn185362] build index on: sledge.policy_assignments properties: { v: 1, key: { endpoint_id: 1, policy_type: 1 }, name: "endpoint_id_1_policy_type_1", ns: "sledge.policy_assignments", sparse: false, background: true } 2015-10-23T22:17:01.567Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:17:12.353Z I INDEX [conn185362] build index on: sledge.policy_assignments properties: { v: 1, key: { endpoint_id: 1 }, name: "endpoint_id_1", ns: "sledge.policy_assignments", sparse: false, background: true } 2015-10-23T22:17:12.353Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:18:51.130Z I INDEX [conn185362] build index on: sledge.event_bases properties: { v: 1, key: { customer_id: 1, severity: 1, reported_at: 1, _type: 1 }, name: "customer_id_1_severity_1_reported_at_1__type_1", ns: "sledge.event_bases", sparse: false, background: true } 2015-10-23T22:18:51.130Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:18:53.549Z I INDEX [conn185362] build index on: sledge.event_bases properties: { v: 1, key: { customer_id: 1, reported_at: -1, _type: 1 }, name: "customer_id_1_reported_at_-1__type_1", ns: "sledge.event_bases", sparse: false, background: true } 2015-10-23T22:18:53.725Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:19:01.386Z I INDEX [conn185362] build index on: sledge.event_bases properties: { v: 1, key: { _type: 1, reported_at: 1 }, name: "_type_1_reported_at_1", ns: "sledge.event_bases", sparse: false, background: true } 2015-10-23T22:19:01.386Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:19:09.029Z I INDEX [conn185362] build index on: sledge.event_bases properties: { v: 1, key: { reported_at: 1 }, name: "reported_at_1", ns: "sledge.event_bases", sparse: false, background: true } 2015-10-23T22:19:09.029Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:19:14.157Z I INDEX [conn185362] build index on: sledge.event_bases properties: { v: 1, key: { customer_id: 1, inserted_at: 1 }, name: "customer_id_1_inserted_at_1", ns: "sledge.event_bases", sparse: true, background: true } 2015-10-23T22:19:14.157Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:19:16.590Z I INDEX [conn185362] build index on: sledge.event_bases properties: { v: 1, key: { endpoint_id: 1, _type: 1, reported_at: 1 }, name: "endpoint_id_1__type_1_reported_at_1", ns: "sledge.event_bases", sparse: false, background: true } 2015-10-23T22:19:16.591Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:19:31.856Z I INDEX [conn185362] build index on: sledge.event_bases properties: { v: 1, key: { user_match_id: 1 }, name: "user_match_id_1", ns: "sledge.event_bases", sparse: true, background: true } 2015-10-23T22:19:31.857Z I INDEX [conn185362] build index done. scanned 0 total records. 0 secs 2015-10-23T22:23:23.141Z I NETWORK [conn185478] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:44562] 2015-10-23T22:34:16.919Z I NETWORK [conn185556] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:44706] 2015-10-23T22:48:22.105Z I NETWORK [conn185654] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:45021] 2015-10-23T22:55:48.625Z I NETWORK [conn185710] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:44983] 2015-10-23T22:56:01.865Z I NETWORK [conn185712] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:45122] 2015-10-23T23:00:04.320Z I NETWORK [conn185728] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:45012] 2015-10-23T23:00:56.222Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-23T23:03:36.179Z I NETWORK [conn185765] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:45084] 2015-10-23T23:23:19.240Z I NETWORK [conn185902] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:45332] 2015-10-23T23:29:14.735Z I NETWORK [conn185941] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:45398] 2015-10-23T23:41:15.419Z I NETWORK [conn186022] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:45675] 2015-10-24T00:05:42.191Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T00:05:50.301Z I NETWORK [conn186199] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:45852] 2015-10-24T00:15:38.577Z I NETWORK [conn186278] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:46128] 2015-10-24T00:27:03.694Z I NETWORK [conn186229] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:45909] 2015-10-24T00:40:48.549Z I NETWORK [conn186452] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:46304] 2015-10-24T00:41:45.137Z I NETWORK [conn186462] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:46320] 2015-10-24T00:44:44.858Z I NETWORK [conn186479] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:46490] 2015-10-24T00:45:38.160Z I NETWORK [conn186487] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:46505] 2015-10-24T00:52:32.293Z I NETWORK [conn186532] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:46596] 2015-10-24T00:56:51.123Z I NETWORK [conn186563] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:46648] 2015-10-24T01:03:59.686Z I NETWORK [conn186608] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:46592] 2015-10-24T01:04:20.000Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T01:04:52.840Z I NETWORK [conn186618] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:46743] 2015-10-24T01:20:44.686Z I NETWORK [conn186610] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:46729] 2015-10-24T01:32:55.541Z I NETWORK [conn186798] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:46929] 2015-10-24T01:35:21.684Z I NETWORK [conn186827] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:47136] 2015-10-24T01:36:45.773Z I NETWORK [conn186837] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:46999] 2015-10-24T01:42:06.269Z I NETWORK [conn186871] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:47064] 2015-10-24T01:45:04.667Z I NETWORK [conn186894] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:47107] 2015-10-24T01:47:47.560Z I NETWORK [conn186911] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:47278] 2015-10-24T01:48:22.810Z I NETWORK [conn186920] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:47300] 2015-10-24T01:50:27.554Z I NETWORK [conn186934] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:47326] 2015-10-24T01:50:57.949Z I NETWORK [conn186935] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:47183] 2015-10-24T01:55:56.094Z I NETWORK [conn186970] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:47395] 2015-10-24T01:58:55.454Z I NETWORK [conn186990] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:47281] 2015-10-24T02:00:21.782Z I NETWORK [conn187005] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:47307] 2015-10-24T02:00:29.068Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T02:00:47.810Z I NETWORK [conn186996] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:47444] 2015-10-24T02:02:11.397Z I NETWORK [conn187014] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:47472] 2015-10-24T02:08:55.803Z I NETWORK [conn187052] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:47396] 2015-10-24T02:13:18.882Z I NETWORK [conn187090] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:47624] 2015-10-24T02:14:20.612Z I NETWORK [conn187098] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:47639] 2015-10-24T02:15:28.960Z I NETWORK [conn187105] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:47652] 2015-10-24T02:15:52.978Z I NETWORK [conn187106] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:47488] 2015-10-24T02:17:00.646Z I NETWORK [conn187117] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:47506] 2015-10-24T02:22:57.770Z I NETWORK [conn187146] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:47571] 2015-10-24T02:30:23.952Z I NETWORK [conn187197] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:47671] 2015-10-24T02:32:01.773Z I NETWORK [conn187206] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:47686] 2015-10-24T02:35:49.154Z I NETWORK [conn187221] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:47723] 2015-10-24T02:37:21.155Z I NETWORK [conn187235] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:47900] 2015-10-24T02:37:52.325Z I NETWORK [conn187243] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:47765] 2015-10-24T02:49:28.203Z I NETWORK [conn187307] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:48033] 2015-10-24T02:56:17.750Z I NETWORK [conn187352] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:47976] 2015-10-24T02:57:22.964Z I NETWORK [conn187367] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:48009] 2015-10-24T03:00:00.540Z I NETWORK [conn187378] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:48034] 2015-10-24T03:00:22.948Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T03:09:56.194Z I NETWORK [conn187431] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:48139] 2015-10-24T03:11:45.470Z I NETWORK [conn187429] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:48267] 2015-10-24T03:21:29.546Z I NETWORK [conn187503] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:48294] 2015-10-24T03:25:46.053Z I NETWORK [conn187508] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:48303] 2015-10-24T03:26:33.258Z I NETWORK [conn187532] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:48479] 2015-10-24T03:29:02.876Z I NETWORK [conn187549] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:48516] 2015-10-24T03:33:35.500Z I NETWORK [conn187550] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:48383] 2015-10-24T03:35:13.878Z I NETWORK [conn187556] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:48530] 2015-10-24T03:42:41.999Z I NETWORK [conn187621] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:48532] 2015-10-24T03:54:08.300Z I NETWORK [conn187673] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:48647] 2015-10-24T03:57:19.367Z I NETWORK [conn187690] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:48684] 2015-10-24T04:00:00.001Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T04:05:29.869Z I NETWORK [conn187725] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:48883] 2015-10-24T04:09:31.315Z I NETWORK [conn187768] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:48848] 2015-10-24T04:21:33.513Z I NETWORK [conn187748] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:48932] 2015-10-24T04:22:16.175Z I NETWORK [conn187838] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:48998] 2015-10-24T04:23:23.174Z I NETWORK [conn187843] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:49006] 2015-10-24T05:00:10.093Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T05:44:53.482Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T05:59:56.388Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T07:00:06.583Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T07:59:55.218Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T08:59:55.144Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T09:59:51.758Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T10:59:53.553Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T11:59:53.376Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T12:59:53.061Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T13:59:51.628Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T14:59:52.554Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T15:59:51.547Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T16:59:51.513Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T17:59:51.971Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T18:59:51.493Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T19:59:51.649Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T20:59:51.410Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T21:59:51.390Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T22:59:51.346Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-24T23:59:51.333Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T00:59:51.340Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T01:59:51.283Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T02:59:51.360Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T03:59:51.217Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T04:59:51.219Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T05:44:51.027Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T05:59:51.184Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T06:59:51.162Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T07:59:51.114Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T08:59:51.111Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T09:59:51.121Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T10:59:51.035Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T11:59:51.001Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T12:59:51.110Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T13:59:51.055Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T14:59:50.922Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T15:59:50.879Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T16:59:50.849Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T17:59:50.818Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T18:59:50.790Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T19:59:50.762Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T20:59:50.736Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T21:59:50.730Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T22:59:50.708Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-25T23:59:50.652Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T00:59:50.649Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T01:59:50.587Z I INDEX [conn170328] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T02:59:52.523Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T03:59:50.539Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T04:59:50.642Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T05:44:50.336Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T05:59:50.479Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T06:59:50.444Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T07:59:51.862Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T08:59:50.470Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T09:59:50.355Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T10:59:50.636Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T11:59:50.333Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T12:59:50.406Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T13:59:50.362Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T14:45:34.965Z I INDEX [conn205508] build index on: scalability.MetaEndpoints properties: { v: 1, key: { cust_id: 1.0 }, name: "cust_id_1", ns: "scalability.MetaEndpoints" } 2015-10-26T14:45:34.965Z I INDEX [conn205508] building index using bulk method 2015-10-26T14:45:37.102Z I - [conn205508] Index Build: 22000/234000 9% 2015-10-26T14:45:40.092Z I - [conn205508] Index Build: 43200/234000 18% 2015-10-26T14:45:43.062Z I - [conn205508] Index Build: 71300/234000 30% 2015-10-26T14:45:46.106Z I - [conn205508] Index Build: 99100/234000 42% 2015-10-26T14:45:49.006Z I - [conn205508] Index Build: 127200/234000 54% 2015-10-26T14:45:52.082Z I - [conn205508] Index Build: 157300/234000 67% 2015-10-26T14:45:55.002Z I - [conn205508] Index Build: 181600/234000 77% 2015-10-26T14:45:58.010Z I - [conn205508] Index Build: 201900/234000 86% 2015-10-26T14:46:01.042Z I - [conn205508] Index Build: 233300/234000 99% 2015-10-26T14:46:29.169Z I INDEX [conn205508] done building bottom layer, going to commit 2015-10-26T14:46:29.694Z I INDEX [conn205508] build index done. scanned 234000 total records. 54 secs 2015-10-26T14:59:54.569Z I INDEX [conn171085] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T15:10:01.952Z I CONTROL [signalProcessingThread] pid=16473 port=27017 64-bit host=ip-172-19-100-23 2015-10-26T15:10:01.952Z I CONTROL [signalProcessingThread] db version v3.0.5 2015-10-26T15:10:01.952Z I CONTROL [signalProcessingThread] git version: 8bc4ae20708dbb493cb09338d9e7be6698e4a3a3 2015-10-26T15:10:01.952Z I CONTROL [signalProcessingThread] build info: Linux ip-10-230-137-3 3.4.43-43.43.amzn1.x86_64 #1 SMP Mon May 6 18:04:41 UTC 2013 x86_64 BOOST_LIB_VERSION=1_49 2015-10-26T15:10:01.952Z I CONTROL [signalProcessingThread] allocator: tcmalloc 2015-10-26T15:10:01.952Z I CONTROL [signalProcessingThread] options: { config: "/etc/mongod.conf", net: { port: 27017, ssl: { PEMKeyFile: "/mongodata/config/mongodb.pem", PEMKeyPassword: "", mode: "preferSSL" } }, operationProfiling: { mode: "off", slowOpThresholdMs: 100 }, processManagement: { fork: true, pidFilePath: "/var/run/mongodb/mongod.pid" }, replication: { oplogSizeMB: 20480, replSetName: "upe_dccb" }, security: { clusterAuthMode: "keyFile", keyFile: "/mongodata/config/keyfile" }, storage: { dbPath: "/mongodata/db", directoryPerDB: true, engine: "wiredTiger", wiredTiger: { collectionConfig: { blockCompressor: "snappy" }, engineConfig: { directoryForIndexes: true, journalCompressor: "snappy", statisticsLogDelaySecs: 0 }, indexConfig: { prefixCompression: true } } }, systemLog: { destination: "file", logAppend: true, logRotate: "reopen", path: "/data/log/mongodb/mongodb.log", timeStampFormat: "iso8601-utc", traceAllExceptions: false, verbosity: 0 } } 2015-10-26T15:14:23.511Z I INDEX [conn206099] build index on: sledge.message_schedules properties: { v: 1, key: { scheduled_at: 1 }, name: "scheduled_at_1", ns: "sledge.message_schedules", background: true } 2015-10-26T15:14:26.118Z I - [conn206099] Index Build (background): 800/123894 0% 2015-10-26T15:14:29.182Z I - [conn206099] Index Build (background): 1500/123994 1% 2015-10-26T15:14:32.134Z I - [conn206099] Index Build (background): 2100/124125 1% 2015-10-26T15:14:35.454Z I - [conn206099] Index Build (background): 3500/124222 2% 2015-10-26T15:14:38.122Z I - [conn206099] Index Build (background): 4100/124273 3% 2015-10-26T15:14:41.360Z I - [conn206099] Index Build (background): 5100/124382 4% 2015-10-26T15:14:44.075Z I - [conn206099] Index Build (background): 5700/124467 4% 2015-10-26T15:14:47.262Z I - [conn206099] Index Build (background): 6600/124591 5% 2015-10-26T15:14:50.014Z I - [conn206099] Index Build (background): 7000/124653 5% 2015-10-26T15:14:53.054Z I - [conn206099] Index Build (background): 8100/124745 6% 2015-10-26T15:14:56.119Z I - [conn206099] Index Build (background): 9400/124844 7% 2015-10-26T15:14:59.177Z I - [conn206099] Index Build (background): 10300/124957 8% 2015-10-26T15:15:02.032Z I - [conn206099] Index Build (background): 11500/125041 9% 2015-10-26T15:15:05.180Z I - [conn206099] Index Build (background): 12500/125103 9% 2015-10-26T15:15:08.278Z I - [conn206099] Index Build (background): 13700/125211 10% 2015-10-26T15:15:11.330Z I - [conn206099] Index Build (background): 14700/125281 11% 2015-10-26T15:15:14.514Z I - [conn206099] Index Build (background): 15800/125344 12% 2015-10-26T15:15:17.074Z I - [conn206099] Index Build (background): 18000/125363 14% 2015-10-26T15:15:20.006Z I - [conn206099] Index Build (background): 25400/125369 20% 2015-10-26T15:15:21.926Z I INDEX [conn206099] build index done. scanned 125369 total records. 58 secs 2015-10-26T15:15:21.995Z I INDEX [conn206099] build index on: sledge.scheduled_messages properties: { v: 1, key: { executed_at: 1, scheduled_at: 1 }, name: "executed_at_1_scheduled_at_1", ns: "sledge.scheduled_messages", background: true } 2015-10-26T15:15:21.995Z I INDEX [conn206099] build index done. scanned 0 total records. 0 secs 2015-10-26T15:15:22.000Z I INDEX [conn206099] build index on: sledge.scheduled_messages properties: { v: 1, key: { scheduled_at: 1, executed_at: 1 }, name: "scheduled_at_1_executed_at_1", ns: "sledge.scheduled_messages", background: true } 2015-10-26T15:15:22.000Z I INDEX [conn206099] build index done. scanned 0 total records. 0 secs 2015-10-26T15:15:22.038Z I INDEX [conn206099] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, info.isSecurityVm: 1 }, name: "customer_id_1_info.isSecurityVm_1", ns: "sledge.endpoints", background: true } 2015-10-26T15:15:25.003Z I - [conn206099] Index Build (background): 199600/215280 92% 2015-10-26T15:15:25.255Z I INDEX [conn206099] build index done. scanned 215280 total records. 3 secs 2015-10-26T15:15:26.515Z I INDEX [conn206131] build index on: sledge.mcs_commands properties: { v: 1, key: { endpoint_id: 1, command_id: 1 }, name: "endpoint_id_1_command_id_1", ns: "sledge.mcs_commands", background: true } 2015-10-26T15:15:26.516Z I INDEX [conn206131] build index done. scanned 0 total records. 0 secs 2015-10-26T15:15:26.595Z I INDEX [conn206172] build index on: sledge.mcs_commands properties: { v: 1, key: { endpoint_id: 1, command_app_id: 1 }, name: "endpoint_id_1_command_app_id_1", ns: "sledge.mcs_commands", background: true } 2015-10-26T15:15:26.595Z I INDEX [conn206172] build index done. scanned 0 total records. 0 secs 2015-10-26T15:15:26.603Z I INDEX [conn206172] build index on: sledge.mcs_commands properties: { v: 1, key: { endpoint_id: 1, posted_app_id: 1, body_sha: 1 }, name: "endpoint_id_1_posted_app_id_1_body_sha_1", ns: "sledge.mcs_commands", background: true } 2015-10-26T15:15:26.604Z I INDEX [conn206172] build index done. scanned 0 total records. 0 secs 2015-10-26T15:15:26.610Z I INDEX [conn206101] build index on: sledge.mcs_commands properties: { v: 1, key: { created_at: 1 }, name: "created_at_1", ns: "sledge.mcs_commands", background: true } 2015-10-26T15:15:26.610Z I INDEX [conn206101] build index done. scanned 0 total records. 0 secs 2015-10-26T15:26:20.363Z I INDEX [conn206164] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, hidden: 1, alert_type: 1 }, name: "customer_id_1_hidden_1_alert_type_1", ns: "sledge.alerts", background: true } 2015-10-26T15:26:20.364Z I INDEX [conn206164] build index done. scanned 14 total records. 0 secs 2015-10-26T15:26:20.368Z I INDEX [conn206164] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, unhide_at: 1 }, name: "customer_id_1_unhide_at_1", ns: "sledge.alerts", background: true } 2015-10-26T15:26:20.368Z I INDEX [conn206164] build index done. scanned 14 total records. 0 secs 2015-10-26T15:26:20.373Z I INDEX [conn206164] build index on: sledge.alerts properties: { v: 1, key: { unhide_at: 1, hidden: 1 }, name: "unhide_at_1_hidden_1", ns: "sledge.alerts", background: true } 2015-10-26T15:26:20.373Z I INDEX [conn206164] build index done. scanned 14 total records. 0 secs 2015-10-26T15:26:20.377Z I INDEX [conn206164] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, data.endpoint_id: 1, alert_type: 1 }, name: "customer_id_1_data.endpoint_id_1_alert_type_1", ns: "sledge.alerts", background: true } 2015-10-26T15:26:20.377Z I INDEX [conn206164] build index done. scanned 14 total records. 0 secs 2015-10-26T15:26:20.381Z I INDEX [conn206164] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, data.threat_id: 1, alert_type: 1 }, name: "customer_id_1_data.threat_id_1_alert_type_1", ns: "sledge.alerts", background: true } 2015-10-26T15:26:20.381Z I INDEX [conn206164] build index done. scanned 14 total records. 0 secs 2015-10-26T15:26:20.385Z I INDEX [conn206164] build index on: sledge.alerts properties: { v: 1, key: { endpoint_id: 1 }, name: "endpoint_id_1", ns: "sledge.alerts", background: true, sparse: true } 2015-10-26T15:26:20.385Z I INDEX [conn206164] build index done. scanned 14 total records. 0 secs 2015-10-26T15:26:21.540Z I INDEX [conn206164] build index on: sledge.endpoint_threats properties: { v: 1, key: { endpoint_id: 1, threat_id: 1 }, name: "endpoint_id_1_threat_id_1", ns: "sledge.endpoint_threats", background: true } 2015-10-26T15:26:21.540Z I INDEX [conn206164] build index done. scanned 0 total records. 0 secs 2015-10-26T15:26:21.549Z I INDEX [conn206142] build index on: sledge.endpoint_threats properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.endpoint_threats", background: true } 2015-10-26T15:26:21.549Z I INDEX [conn206142] build index done. scanned 0 total records. 0 secs 2015-10-26T15:26:21.600Z I INDEX [conn206142] build index on: sledge.event_bases properties: { v: 1, key: { threat_id: 1, _type: 1 }, name: "threat_id_1__type_1", ns: "sledge.event_bases", background: true } 2015-10-26T15:26:23.924Z I INDEX [conn206226] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.report_bases", background: true } 2015-10-26T15:26:23.924Z I INDEX [conn206226] build index done. scanned 6 total records. 0 secs 2015-10-26T15:26:23.930Z I INDEX [conn206226] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1 }, name: "customer_id_1__type_1", ns: "sledge.report_bases", background: true } 2015-10-26T15:26:23.930Z I INDEX [conn206226] build index done. scanned 6 total records. 0 secs 2015-10-26T15:26:23.936Z I INDEX [conn206226] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, tm: 1 }, name: "customer_id_1__type_1_tm_1", ns: "sledge.report_bases", background: true } 2015-10-26T15:26:23.937Z I INDEX [conn206226] build index done. scanned 6 total records. 0 secs 2015-10-26T15:26:23.943Z I INDEX [conn206226] build index on: sledge.report_bases properties: { v: 1, key: { ham_idx_upd: -1 }, name: "ham_idx_upd_-1", ns: "sledge.report_bases", background: true, sparse: true } 2015-10-26T15:26:23.943Z I INDEX [conn206226] build index done. scanned 6 total records. 0 secs 2015-10-26T15:26:23.951Z I INDEX [conn206226] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, name: 1, group: 1 }, name: "customer_id_1_name_1_group_1", ns: "sledge.report_bases", background: true } 2015-10-26T15:26:23.951Z I INDEX [conn206226] build index done. scanned 6 total records. 0 secs 2015-10-26T15:26:23.957Z I INDEX [conn206226] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, hits.type: 1, name: 1, group: 1 }, name: "customer_id_1_hits.type_1_name_1_group_1", ns: "sledge.report_bases", background: true } 2015-10-26T15:26:23.957Z I INDEX [conn206226] build index done. scanned 6 total records. 0 secs 2015-10-26T15:26:23.965Z I INDEX [conn206235] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, name: 1 }, name: "customer_id_1__type_1_name_1", ns: "sledge.report_bases", background: true } 2015-10-26T15:26:23.965Z I INDEX [conn206235] build index done. scanned 6 total records. 0 secs 2015-10-26T15:26:24.003Z I - [conn206142] Index Build (background): 161500/215318 75% 2015-10-26T15:26:24.768Z I INDEX [conn206142] build index done. scanned 215326 total records. 3 secs 2015-10-26T15:26:25.428Z I INDEX [conn206229] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, dom: 1 }, name: "customer_id_1_dom_1", ns: "sledge.report_bases", background: true } 2015-10-26T15:26:25.429Z I INDEX [conn206229] build index done. scanned 12 total records. 0 secs 2015-10-26T15:26:25.434Z I INDEX [conn206235] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, cat: 1 }, name: "customer_id_1_cat_1", ns: "sledge.report_bases", background: true } 2015-10-26T15:26:25.434Z I INDEX [conn206235] build index done. scanned 12 total records. 0 secs 2015-10-26T15:26:25.438Z I INDEX [conn206235] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, rsn: 1 }, name: "customer_id_1_rsn_1", ns: "sledge.report_bases", background: true } 2015-10-26T15:26:25.439Z I INDEX [conn206235] build index done. scanned 12 total records. 0 secs 2015-10-26T15:26:25.444Z I INDEX [conn206235] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, hits.type: 1, dom: 1 }, name: "customer_id_1_hits.type_1_dom_1", ns: "sledge.report_bases", background: true } 2015-10-26T15:26:25.444Z I INDEX [conn206235] build index done. scanned 12 total records. 0 secs 2015-10-26T15:26:25.448Z I INDEX [conn206235] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, dom: 1, cat: 1, rsn: 1 }, name: "customer_id_1__type_1_dom_1_cat_1_rsn_1", ns: "sledge.report_bases", background: true } 2015-10-26T15:26:25.449Z I INDEX [conn206235] build index done. scanned 12 total records. 0 secs 2015-10-26T15:26:25.463Z I INDEX [conn206235] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, usr: 1 }, name: "customer_id_1_usr_1", ns: "sledge.report_bases", background: true } 2015-10-26T15:26:25.464Z I INDEX [conn206235] build index done. scanned 13 total records. 0 secs 2015-10-26T15:26:25.468Z I INDEX [conn206235] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, loc: 1 }, name: "customer_id_1_loc_1", ns: "sledge.report_bases", background: true } 2015-10-26T15:26:25.468Z I INDEX [conn206235] build index done. scanned 13 total records. 0 secs 2015-10-26T15:26:25.472Z I INDEX [conn206235] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, hits.type: 1, usr: 1 }, name: "customer_id_1_hits.type_1_usr_1", ns: "sledge.report_bases", background: true } 2015-10-26T15:26:25.473Z I INDEX [conn206235] build index done. scanned 13 total records. 0 secs 2015-10-26T15:26:25.477Z I INDEX [conn206235] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, usr: 1, loc: 1 }, name: "customer_id_1__type_1_usr_1_loc_1", ns: "sledge.report_bases", background: true } 2015-10-26T15:26:25.477Z I INDEX [conn206235] build index done. scanned 13 total records. 0 secs 2015-10-26T15:26:26.046Z I INDEX [conn206235] build index on: sledge.endpoint_devices properties: { v: 1, key: { customer_id: 1, detected_at: 1 }, name: "customer_id_1_detected_at_1", ns: "sledge.endpoint_devices", background: true } 2015-10-26T15:26:26.046Z I INDEX [conn206235] build index done. scanned 2 total records. 0 secs 2015-10-26T15:26:26.052Z I INDEX [conn206235] build index on: sledge.endpoint_devices properties: { v: 1, key: { endpoint_id: 1, device_id: 1, instance_id: 1 }, name: "endpoint_id_1_device_id_1_instance_id_1", ns: "sledge.endpoint_devices", background: true } 2015-10-26T15:26:26.053Z I INDEX [conn206235] build index done. scanned 2 total records. 0 secs 2015-10-26T15:26:26.057Z I INDEX [conn206235] build index on: sledge.endpoint_devices properties: { v: 1, key: { endpoint_id: 1, device_id: 1, device_type: 1, instance_id: 1, description: 1, detected_at: 1 }, name: "endpoint_id_1_device_id_1_device_type_1_instance_id_1_description_1_detected_at_1", ns: "sledge.endpoint_devices", background: true } 2015-10-26T15:26:26.057Z I INDEX [conn206235] build index done. scanned 2 total records. 0 secs 2015-10-26T15:26:26.061Z I INDEX [conn206235] build index on: sledge.endpoint_devices properties: { v: 1, key: { customer_id: 1, device_type: 1, detected_at: 1 }, name: "customer_id_1_device_type_1_detected_at_1", ns: "sledge.endpoint_devices", background: true } 2015-10-26T15:26:26.061Z I INDEX [conn206235] build index done. scanned 2 total records. 0 secs 2015-10-26T15:26:26.097Z I INDEX [conn206229] build index on: sledge.event_bases properties: { v: 1, key: { device_id: 1, _type: 1, reported_at: 1 }, name: "device_id_1__type_1_reported_at_1", ns: "sledge.event_bases", background: true } 2015-10-26T15:26:29.003Z I - [conn206229] Index Build (background): 165500/215357 76% 2015-10-26T15:26:29.732Z I INDEX [conn206229] build index done. scanned 215362 total records. 3 secs 2015-10-26T15:26:29.737Z I INDEX [conn206229] build index on: sledge.event_bases properties: { v: 1, key: { device_id: 1, reported_at: -1 }, name: "device_id_1_reported_at_-1", ns: "sledge.event_bases", background: true } 2015-10-26T15:26:32.002Z I - [conn206229] Index Build (background): 166200/215372 77% 2015-10-26T15:26:32.671Z I INDEX [conn206229] build index done. scanned 215376 total records. 2 secs 2015-10-26T15:26:32.677Z I INDEX [conn206229] build index on: sledge.event_bases properties: { v: 1, key: { customer_id: 1, device_id: 1 }, name: "customer_id_1_device_id_1", ns: "sledge.event_bases", background: true } 2015-10-26T15:26:35.002Z I - [conn206229] Index Build (background): 164700/215390 76% 2015-10-26T15:26:35.716Z I INDEX [conn206229] build index done. scanned 215394 total records. 3 secs 2015-10-26T15:59:50.228Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T16:59:52.465Z I INDEX [conn170327] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T17:59:50.121Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T18:00:57.749Z I INDEX [conn207305] build index on: test.rendered_policies properties: { v: 1, key: { policy_fragment_ids: 1 }, name: "policy_fragment_ids_1", ns: "test.rendered_policies", sparse: false, background: true } 2015-10-26T18:00:57.749Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:57.938Z I INDEX [conn207305] build index on: test.customers properties: { v: 1, key: { licenses.expires_on: 1 }, name: "licenses.expires_on_1", ns: "test.customers", sparse: false, background: true } 2015-10-26T18:00:57.938Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:57.945Z I INDEX [conn207305] build index on: test.customers properties: { v: 1, key: { beta: 1 }, name: "beta_1", ns: "test.customers", sparse: true, background: true } 2015-10-26T18:00:57.945Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.008Z I INDEX [conn207305] build index on: test.directory_objects properties: { v: 1, key: { c: 1, _type: 1 }, name: "c_1__type_1", ns: "test.directory_objects", sparse: false, background: true } 2015-10-26T18:00:58.008Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.011Z I INDEX [conn207305] build index on: test.directory_objects properties: { v: 1, key: { c: 1, _type: 1, name_lowercase: 1 }, name: "c_1__type_1_name_lowercase_1", ns: "test.directory_objects", sparse: false, background: true } 2015-10-26T18:00:58.012Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.015Z I INDEX [conn207305] build index on: test.directory_objects properties: { v: 1, key: { c: 1, _type: 1, search_tokens: 1 }, name: "c_1__type_1_search_tokens_1", ns: "test.directory_objects", sparse: false, background: true } 2015-10-26T18:00:58.015Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.020Z I INDEX [conn207305] build index on: test.directory_objects properties: { v: 1, key: { c: 1, _type: 1, external_id: 1 }, name: "c_1__type_1_external_id_1", ns: "test.directory_objects", sparse: false, background: true } 2015-10-26T18:00:58.020Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.024Z I INDEX [conn207305] build index on: test.directory_objects properties: { v: 1, key: { c: 1, _type: 1, origin: 1 }, name: "c_1__type_1_origin_1", ns: "test.directory_objects", sparse: false, background: true } 2015-10-26T18:00:58.024Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.028Z I INDEX [conn207305] build index on: test.directory_objects properties: { v: 1, key: { c: 1, _type: 1, email: 1 }, name: "c_1__type_1_email_1", ns: "test.directory_objects", sparse: false, background: true } 2015-10-26T18:00:58.028Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.032Z I INDEX [conn207305] build index on: test.directory_objects properties: { v: 1, key: { c: 1, _type: 1, dn: 1 }, name: "c_1__type_1_dn_1", ns: "test.directory_objects", sparse: false, background: true } 2015-10-26T18:00:58.032Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.192Z I INDEX [conn207305] build index on: test.policies properties: { v: 1, key: { customer_id: 1, target: 1, priority: -1 }, name: "customer_id_1_target_1_priority_-1", ns: "test.policies", sparse: false, background: true } 2015-10-26T18:00:58.192Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.196Z I INDEX [conn207305] build index on: test.policies properties: { v: 1, key: { enabled: 1, disable_at: 1 }, name: "enabled_1_disable_at_1", ns: "test.policies", sparse: false, background: true } 2015-10-26T18:00:58.196Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.457Z I INDEX [conn207305] build index on: test.mcs_policy_bases properties: { v: 1, key: { customer_id: 1, _type: 1 }, name: "customer_id_1__type_1", ns: "test.mcs_policy_bases", sparse: false, background: true } 2015-10-26T18:00:58.457Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.499Z I INDEX [conn207305] build index on: test.global_exclusions properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "test.global_exclusions", sparse: false, background: true } 2015-10-26T18:00:58.499Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.517Z I INDEX [conn207305] build index on: test.customer_flags properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "test.customer_flags", sparse: false, background: true } 2015-10-26T18:00:58.517Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.553Z I INDEX [conn207305] build index on: test.deployment_tokens properties: { v: 1, unique: true, key: { registration_token: 1 }, name: "registration_token_1", ns: "test.deployment_tokens", sparse: false, background: true } 2015-10-26T18:00:58.553Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.556Z I INDEX [conn207305] build index on: test.deployment_tokens properties: { v: 1, key: { user_id: 1 }, name: "user_id_1", ns: "test.deployment_tokens", sparse: false, background: true } 2015-10-26T18:00:58.557Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.560Z I INDEX [conn207305] build index on: test.deployment_tokens properties: { v: 1, key: { customer_id: 1, user_id: 1, beta: 1 }, name: "customer_id_1_user_id_1_beta_1", ns: "test.deployment_tokens", sparse: false, background: true } 2015-10-26T18:00:58.560Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.614Z I INDEX [conn207305] build index on: test.directory_user_matches properties: { v: 1, key: { local_endpoint_id: 1 }, name: "local_endpoint_id_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-26T18:00:58.614Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.618Z I INDEX [conn207305] build index on: test.directory_user_matches properties: { v: 1, key: { name_lowercase: 1 }, name: "name_lowercase_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-26T18:00:58.618Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.622Z I INDEX [conn207305] build index on: test.directory_user_matches properties: { v: 1, unique: true, key: { name_lowercase: 1, c: 1 }, name: "name_lowercase_1_c_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-26T18:00:58.622Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.626Z I INDEX [conn207305] build index on: test.directory_user_matches properties: { v: 1, key: { user_id: 1 }, name: "user_id_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-26T18:00:58.626Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.630Z I INDEX [conn207305] build index on: test.directory_user_matches properties: { v: 1, key: { customer_id: 1, name_lowercase: 1 }, name: "customer_id_1_name_lowercase_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-26T18:00:58.630Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.634Z I INDEX [conn207305] build index on: test.directory_user_matches properties: { v: 1, key: { c: 1, local_endpoint_id: 1 }, name: "c_1_local_endpoint_id_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-26T18:00:58.634Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.640Z I INDEX [conn207305] build index on: test.directory_user_matches properties: { v: 1, key: { c: 1, name_lowercase: 1 }, name: "c_1_name_lowercase_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-26T18:00:58.640Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.645Z I INDEX [conn207305] build index on: test.directory_user_matches properties: { v: 1, key: { c: 1, external_id: 1 }, name: "c_1_external_id_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-26T18:00:58.645Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.649Z I INDEX [conn207305] build index on: test.directory_user_matches properties: { v: 1, key: { c: 1, origin: 1 }, name: "c_1_origin_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-26T18:00:58.649Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.654Z I INDEX [conn207305] build index on: test.directory_user_matches properties: { v: 1, key: { c: 1, name: 1 }, name: "c_1_name_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-26T18:00:58.654Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.659Z I INDEX [conn207305] build index on: test.directory_user_matches properties: { v: 1, key: { c: 1, dn: 1 }, name: "c_1_dn_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-26T18:00:58.659Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.695Z I INDEX [conn207305] build index on: test.endpoints properties: { v: 1, key: { user_match_id: 1 }, name: "user_match_id_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-26T18:00:58.695Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.699Z I INDEX [conn207305] build index on: test.endpoints properties: { v: 1, key: { domain_name: 1, computer_name: 1, operating_system: 1 }, name: "domain_name_1_computer_name_1_operating_system_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-26T18:00:58.699Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.703Z I INDEX [conn207305] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-26T18:00:58.703Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.706Z I INDEX [conn207305] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, feature_codes: 1 }, name: "customer_id_1_feature_codes_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-26T18:00:58.707Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.710Z I INDEX [conn207305] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, label: 1 }, name: "customer_id_1_label_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-26T18:00:58.710Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.714Z I INDEX [conn207305] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, label_lowercase: 1 }, name: "customer_id_1_label_lowercase_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-26T18:00:58.714Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.720Z I INDEX [conn207305] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, user_match_id: 1 }, name: "customer_id_1_user_match_id_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-26T18:00:58.720Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.724Z I INDEX [conn207305] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, search_tokens: 1 }, name: "customer_id_1_search_tokens_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-26T18:00:58.724Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.728Z I INDEX [conn207305] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, transport: 1 }, name: "customer_id_1_transport_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-26T18:00:58.728Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.732Z I INDEX [conn207305] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, endpoint_type: 1 }, name: "customer_id_1_endpoint_type_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-26T18:00:58.732Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.737Z I INDEX [conn207305] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, info.platform: 1 }, name: "customer_id_1_info.platform_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-26T18:00:58.737Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.741Z I INDEX [conn207305] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, registered_at: 1 }, name: "customer_id_1_registered_at_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-26T18:00:58.742Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.747Z I INDEX [conn207305] build index on: test.endpoints properties: { v: 1, key: { last_connected_from: 1 }, name: "last_connected_from_1", ns: "test.endpoints", sparse: true, background: true } 2015-10-26T18:00:58.747Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.754Z I INDEX [conn207305] build index on: test.endpoints properties: { v: 1, key: { beta: 1 }, name: "beta_1", ns: "test.endpoints", sparse: true, background: true } 2015-10-26T18:00:58.754Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.848Z I INDEX [conn207305] build index on: test.policy_assignments properties: { v: 1, key: { endpoint_id: 1, policy_type: 1 }, name: "endpoint_id_1_policy_type_1", ns: "test.policy_assignments", sparse: false, background: true } 2015-10-26T18:00:58.848Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.851Z I INDEX [conn207305] build index on: test.policy_assignments properties: { v: 1, key: { endpoint_id: 1 }, name: "endpoint_id_1", ns: "test.policy_assignments", sparse: false, background: true } 2015-10-26T18:00:58.851Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.892Z I INDEX [conn207305] build index on: test.event_bases properties: { v: 1, key: { customer_id: 1, severity: 1, reported_at: 1, _type: 1 }, name: "customer_id_1_severity_1_reported_at_1__type_1", ns: "test.event_bases", sparse: false, background: true } 2015-10-26T18:00:58.892Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.895Z I INDEX [conn207305] build index on: test.event_bases properties: { v: 1, key: { customer_id: 1, reported_at: -1, _type: 1 }, name: "customer_id_1_reported_at_-1__type_1", ns: "test.event_bases", sparse: false, background: true } 2015-10-26T18:00:58.895Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.899Z I INDEX [conn207305] build index on: test.event_bases properties: { v: 1, key: { _type: 1, reported_at: 1 }, name: "_type_1_reported_at_1", ns: "test.event_bases", sparse: false, background: true } 2015-10-26T18:00:58.899Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.903Z I INDEX [conn207305] build index on: test.event_bases properties: { v: 1, key: { reported_at: 1 }, name: "reported_at_1", ns: "test.event_bases", sparse: false, background: true } 2015-10-26T18:00:58.903Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.906Z I INDEX [conn207305] build index on: test.event_bases properties: { v: 1, key: { customer_id: 1, inserted_at: 1 }, name: "customer_id_1_inserted_at_1", ns: "test.event_bases", sparse: true, background: true } 2015-10-26T18:00:58.907Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.913Z I INDEX [conn207305] build index on: test.event_bases properties: { v: 1, key: { endpoint_id: 1, _type: 1, reported_at: 1 }, name: "endpoint_id_1__type_1_reported_at_1", ns: "test.event_bases", sparse: false, background: true } 2015-10-26T18:00:58.913Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:00:58.917Z I INDEX [conn207305] build index on: test.event_bases properties: { v: 1, key: { user_match_id: 1 }, name: "user_match_id_1", ns: "test.event_bases", sparse: true, background: true } 2015-10-26T18:00:58.917Z I INDEX [conn207305] build index done. scanned 0 total records. 0 secs 2015-10-26T18:59:50.857Z I INDEX [conn206675] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T19:12:37.821Z I INDEX [conn207756] build index on: sledge.deployment_tokens properties: { v: 1, key: { customer_id: 1, user_id: 1, beta: 1 }, name: "customer_id_1_user_id_1_beta_1", ns: "sledge.deployment_tokens", sparse: false, background: true } 2015-10-26T19:12:37.821Z I INDEX [conn207756] build index done. scanned 0 total records. 0 secs 2015-10-26T19:59:50.280Z I INDEX [conn206675] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T20:09:04.670Z I INDEX [conn208043] build index on: sledge.deployment_tokens properties: { v: 1, key: { customer_id: 1, user_id: 1, beta: 1 }, name: "customer_id_1_user_id_1_beta_1", ns: "sledge.deployment_tokens", sparse: false, background: true } 2015-10-26T20:09:04.670Z I INDEX [conn208043] build index done. scanned 1 total records. 0 secs 2015-10-26T20:59:50.304Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T21:59:51.728Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T22:59:49.981Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-26T23:59:49.961Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T00:59:51.463Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T01:59:49.886Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T02:59:51.319Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T03:59:50.430Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T04:59:51.309Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T05:44:49.620Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T05:59:52.111Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T06:59:49.766Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T07:59:51.170Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T08:59:51.547Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T09:59:49.919Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T10:59:50.467Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T11:59:49.683Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T12:59:49.765Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T13:59:50.518Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T14:45:57.402Z I INDEX [conn213784] build index on: sledge.rendered_policies properties: { v: 1, key: { policy_fragment_ids: 1 }, name: "policy_fragment_ids_1", ns: "sledge.rendered_policies", sparse: false, background: true } 2015-10-27T14:45:57.402Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:57.579Z I INDEX [conn213784] build index on: sledge.customers properties: { v: 1, key: { licenses.expires_on: 1 }, name: "licenses.expires_on_1", ns: "sledge.customers", sparse: false, background: true } 2015-10-27T14:45:57.579Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:57.583Z I INDEX [conn213784] build index on: sledge.customers properties: { v: 1, key: { beta: 1 }, name: "beta_1", ns: "sledge.customers", sparse: true, background: true } 2015-10-27T14:45:57.583Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:57.646Z I INDEX [conn213784] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1 }, name: "c_1__type_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-27T14:45:57.646Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:57.650Z I INDEX [conn213784] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, name_lowercase: 1 }, name: "c_1__type_1_name_lowercase_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-27T14:45:57.650Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:57.654Z I INDEX [conn213784] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, search_tokens: 1 }, name: "c_1__type_1_search_tokens_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-27T14:45:57.654Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:57.659Z I INDEX [conn213784] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, external_id: 1 }, name: "c_1__type_1_external_id_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-27T14:45:57.659Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:57.663Z I INDEX [conn213784] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, origin: 1 }, name: "c_1__type_1_origin_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-27T14:45:57.664Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:57.668Z I INDEX [conn213784] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, email: 1 }, name: "c_1__type_1_email_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-27T14:45:57.668Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:57.672Z I INDEX [conn213784] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, dn: 1 }, name: "c_1__type_1_dn_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-27T14:45:57.672Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:57.829Z I INDEX [conn213784] build index on: sledge.policies properties: { v: 1, key: { customer_id: 1, target: 1, priority: -1 }, name: "customer_id_1_target_1_priority_-1", ns: "sledge.policies", sparse: false, background: true } 2015-10-27T14:45:57.829Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:57.833Z I INDEX [conn213784] build index on: sledge.policies properties: { v: 1, key: { enabled: 1, disable_at: 1 }, name: "enabled_1_disable_at_1", ns: "sledge.policies", sparse: false, background: true } 2015-10-27T14:45:57.833Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.135Z I INDEX [conn213784] build index on: sledge.mcs_policy_bases properties: { v: 1, key: { customer_id: 1, _type: 1 }, name: "customer_id_1__type_1", ns: "sledge.mcs_policy_bases", sparse: false, background: true } 2015-10-27T14:45:58.135Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.182Z I INDEX [conn213784] build index on: sledge.global_exclusions properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.global_exclusions", sparse: false, background: true } 2015-10-27T14:45:58.182Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.200Z I INDEX [conn213784] build index on: sledge.customer_flags properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.customer_flags", sparse: false, background: true } 2015-10-27T14:45:58.200Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.323Z I INDEX [conn213784] build index on: sledge.directory_user_matches properties: { v: 1, key: { local_endpoint_id: 1 }, name: "local_endpoint_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-27T14:45:58.324Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.327Z I INDEX [conn213784] build index on: sledge.directory_user_matches properties: { v: 1, key: { name_lowercase: 1 }, name: "name_lowercase_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-27T14:45:58.327Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.334Z I INDEX [conn213784] build index on: sledge.directory_user_matches properties: { v: 1, unique: true, key: { name_lowercase: 1, c: 1 }, name: "name_lowercase_1_c_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-27T14:45:58.334Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.337Z I INDEX [conn213784] build index on: sledge.directory_user_matches properties: { v: 1, key: { user_id: 1 }, name: "user_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-27T14:45:58.337Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.344Z I INDEX [conn213784] build index on: sledge.directory_user_matches properties: { v: 1, key: { customer_id: 1, name_lowercase: 1 }, name: "customer_id_1_name_lowercase_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-27T14:45:58.344Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.348Z I INDEX [conn213784] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, local_endpoint_id: 1 }, name: "c_1_local_endpoint_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-27T14:45:58.348Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.352Z I INDEX [conn213784] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, name_lowercase: 1 }, name: "c_1_name_lowercase_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-27T14:45:58.352Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.356Z I INDEX [conn213784] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, external_id: 1 }, name: "c_1_external_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-27T14:45:58.356Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.360Z I INDEX [conn213784] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, origin: 1 }, name: "c_1_origin_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-27T14:45:58.360Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.364Z I INDEX [conn213784] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, name: 1 }, name: "c_1_name_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-27T14:45:58.364Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.367Z I INDEX [conn213784] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, dn: 1 }, name: "c_1_dn_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-27T14:45:58.367Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.405Z I INDEX [conn213784] build index on: sledge.endpoints properties: { v: 1, key: { user_match_id: 1 }, name: "user_match_id_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-27T14:45:58.405Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.409Z I INDEX [conn213784] build index on: sledge.endpoints properties: { v: 1, key: { domain_name: 1, computer_name: 1, operating_system: 1 }, name: "domain_name_1_computer_name_1_operating_system_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-27T14:45:58.409Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.412Z I INDEX [conn213784] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-27T14:45:58.412Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.416Z I INDEX [conn213784] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, feature_codes: 1 }, name: "customer_id_1_feature_codes_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-27T14:45:58.416Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.419Z I INDEX [conn213784] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, label: 1 }, name: "customer_id_1_label_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-27T14:45:58.419Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.423Z I INDEX [conn213784] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, label_lowercase: 1 }, name: "customer_id_1_label_lowercase_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-27T14:45:58.423Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.427Z I INDEX [conn213784] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, user_match_id: 1 }, name: "customer_id_1_user_match_id_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-27T14:45:58.427Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.430Z I INDEX [conn213784] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, search_tokens: 1 }, name: "customer_id_1_search_tokens_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-27T14:45:58.430Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.436Z I INDEX [conn213784] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, transport: 1 }, name: "customer_id_1_transport_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-27T14:45:58.436Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.439Z I INDEX [conn213784] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, endpoint_type: 1 }, name: "customer_id_1_endpoint_type_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-27T14:45:58.439Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.443Z I INDEX [conn213784] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, info.platform: 1 }, name: "customer_id_1_info.platform_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-27T14:45:58.443Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.448Z I INDEX [conn213784] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, registered_at: 1 }, name: "customer_id_1_registered_at_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-27T14:45:58.448Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.452Z I INDEX [conn213784] build index on: sledge.endpoints properties: { v: 1, key: { last_connected_from: 1 }, name: "last_connected_from_1", ns: "sledge.endpoints", sparse: true, background: true } 2015-10-27T14:45:58.452Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.456Z I INDEX [conn213784] build index on: sledge.endpoints properties: { v: 1, key: { beta: 1 }, name: "beta_1", ns: "sledge.endpoints", sparse: true, background: true } 2015-10-27T14:45:58.456Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.560Z I INDEX [conn213784] build index on: sledge.policy_assignments properties: { v: 1, key: { endpoint_id: 1, policy_type: 1 }, name: "endpoint_id_1_policy_type_1", ns: "sledge.policy_assignments", sparse: false, background: true } 2015-10-27T14:45:58.560Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:45:58.563Z I INDEX [conn213784] build index on: sledge.policy_assignments properties: { v: 1, key: { endpoint_id: 1 }, name: "endpoint_id_1", ns: "sledge.policy_assignments", sparse: false, background: true } 2015-10-27T14:45:58.563Z I INDEX [conn213784] build index done. scanned 0 total records. 0 secs 2015-10-27T14:59:49.708Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T15:59:49.615Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T16:59:49.568Z I INDEX [conn177273] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T17:59:50.821Z I INDEX [conn183533] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T18:55:56.475Z I NETWORK [conn213987] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:47812] 2015-10-27T18:58:47.190Z I NETWORK [conn215212] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:50208] 2015-10-27T18:58:47.343Z I NETWORK [conn215211] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:50296] 2015-10-27T19:00:56.451Z I INDEX [conn215177] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T19:02:36.911Z I NETWORK [conn215235] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:50241] 2015-10-27T19:10:01.599Z I CONTROL [signalProcessingThread] pid=16473 port=27017 64-bit host=ip-172-19-100-23 2015-10-27T19:10:01.599Z I CONTROL [signalProcessingThread] db version v3.0.5 2015-10-27T19:10:01.599Z I CONTROL [signalProcessingThread] git version: 8bc4ae20708dbb493cb09338d9e7be6698e4a3a3 2015-10-27T19:10:01.599Z I CONTROL [signalProcessingThread] build info: Linux ip-10-230-137-3 3.4.43-43.43.amzn1.x86_64 #1 SMP Mon May 6 18:04:41 UTC 2013 x86_64 BOOST_LIB_VERSION=1_49 2015-10-27T19:10:01.599Z I CONTROL [signalProcessingThread] allocator: tcmalloc 2015-10-27T19:10:01.599Z I CONTROL [signalProcessingThread] options: { config: "/etc/mongod.conf", net: { port: 27017, ssl: { PEMKeyFile: "/mongodata/config/mongodb.pem", PEMKeyPassword: "", mode: "preferSSL" } }, operationProfiling: { mode: "off", slowOpThresholdMs: 100 }, processManagement: { fork: true, pidFilePath: "/var/run/mongodb/mongod.pid" }, replication: { oplogSizeMB: 20480, replSetName: "upe_dccb" }, security: { clusterAuthMode: "keyFile", keyFile: "/mongodata/config/keyfile" }, storage: { dbPath: "/mongodata/db", directoryPerDB: true, engine: "wiredTiger", wiredTiger: { collectionConfig: { blockCompressor: "snappy" }, engineConfig: { directoryForIndexes: true, journalCompressor: "snappy", statisticsLogDelaySecs: 0 }, indexConfig: { prefixCompression: true } } }, systemLog: { destination: "file", logAppend: true, logRotate: "reopen", path: "/data/log/mongodb/mongodb.log", timeStampFormat: "iso8601-utc", traceAllExceptions: false, verbosity: 0 } } 2015-10-27T19:11:22.606Z I NETWORK [conn215236] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:50329] 2015-10-27T19:23:41.907Z I NETWORK [conn215372] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:50531] 2015-10-27T19:26:51.759Z I NETWORK [conn215446] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:50525] 2015-10-27T19:26:52.244Z I NETWORK [conn215445] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:50611] 2015-10-27T19:36:15.920Z I NETWORK [conn215482] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:50648] 2015-10-27T19:38:48.178Z I NETWORK [conn215481] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:50561] 2015-10-27T19:50:33.102Z I NETWORK [conn215711] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:50819] 2015-10-27T19:50:34.800Z I NETWORK [conn215575] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:50782] 2015-10-27T19:54:28.534Z I NETWORK [conn215724] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:50917] 2015-10-27T19:54:29.989Z I NETWORK [conn215722] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:50827] 2015-10-27T19:59:06.916Z I NETWORK [conn215779] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:51004] 2015-10-27T19:59:07.663Z I NETWORK [conn215778] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:50911] 2015-10-27T20:07:05.034Z I INDEX [conn215733] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T20:07:05.070Z I INDEX [conn215819] build index on: sledge.scheduled_messages properties: { v: 1, key: { executed_at: 1, scheduled_at: 1 }, name: "executed_at_1_scheduled_at_1", ns: "sledge.scheduled_messages", background: true } 2015-10-27T20:07:05.071Z I INDEX [conn215819] build index done. scanned 0 total records. 0 secs 2015-10-27T20:07:06.424Z I INDEX [conn215843] build index on: sledge.scheduled_messages properties: { v: 1, key: { scheduled_at: 1, executed_at: 1 }, name: "scheduled_at_1_executed_at_1", ns: "sledge.scheduled_messages", background: true } 2015-10-27T20:07:06.472Z I INDEX [conn215843] build index done. scanned 0 total records. 0 secs 2015-10-27T20:07:17.641Z I INDEX [conn215820] build index on: sledge.mcs_commands properties: { v: 1, key: { endpoint_id: 1, command_id: 1 }, name: "endpoint_id_1_command_id_1", ns: "sledge.mcs_commands", background: true } 2015-10-27T20:07:18.140Z I INDEX [conn215820] build index done. scanned 5363 total records. 0 secs 2015-10-27T20:07:52.754Z I INDEX [conn215844] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, info.isSecurityVm: 1 }, name: "customer_id_1_info.isSecurityVm_1", ns: "sledge.endpoints", background: true } 2015-10-27T20:07:52.945Z I INDEX [conn215820] build index on: sledge.mcs_commands properties: { v: 1, key: { endpoint_id: 1, command_app_id: 1 }, name: "endpoint_id_1_command_app_id_1", ns: "sledge.mcs_commands", background: true } 2015-10-27T20:07:54.161Z I INDEX [conn215820] build index done. scanned 8052 total records. 1 secs 2015-10-27T20:07:54.330Z I INDEX [conn215820] build index on: sledge.mcs_commands properties: { v: 1, key: { endpoint_id: 1, posted_app_id: 1, body_sha: 1 }, name: "endpoint_id_1_posted_app_id_1_body_sha_1", ns: "sledge.mcs_commands", background: true } 2015-10-27T20:07:54.852Z I - [conn170338] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:07:54.958Z I CONTROL [conn170338] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BF297"},{"b":"400000","o":"56EF14"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo16getExecutorCountEPNS_16OperationContextEPNS_10CollectionERKNS_12CountRequestENS_12PlanExecutor11YieldPolicyEPPS7_+0x3D7) [0xbbf297] mongod(_ZN5mongo8CmdCount3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x134) [0x96ef14] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:07:55.003Z I - [conn215844] Index Build (background): 18600/165692 11% 2015-10-27T20:07:55.553Z I INDEX [conn215820] build index done. scanned 10358 total records. 1 secs 2015-10-27T20:07:55.754Z I INDEX [conn215820] build index on: sledge.mcs_commands properties: { v: 1, key: { created_at: 1 }, name: "created_at_1", ns: "sledge.mcs_commands", background: true } 2015-10-27T20:07:57.478Z I INDEX [conn215820] build index done. scanned 13619 total records. 1 secs 2015-10-27T20:07:58.003Z I - [conn215844] Index Build (background): 46200/165692 27% 2015-10-27T20:07:58.688Z I INDEX [conn215819] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, hidden: 1, alert_type: 1 }, name: "customer_id_1_hidden_1_alert_type_1", ns: "sledge.alerts", background: true } 2015-10-27T20:07:58.689Z I INDEX [conn215819] build index done. scanned 0 total records. 0 secs 2015-10-27T20:07:58.851Z I INDEX [conn215819] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, unhide_at: 1 }, name: "customer_id_1_unhide_at_1", ns: "sledge.alerts", background: true } 2015-10-27T20:07:58.851Z I INDEX [conn215819] build index done. scanned 0 total records. 0 secs 2015-10-27T20:08:01.005Z I - [conn215844] Index Build (background): 100500/165692 60% 2015-10-27T20:08:02.357Z I INDEX [conn215819] build index on: sledge.alerts properties: { v: 1, key: { unhide_at: 1, hidden: 1 }, name: "unhide_at_1_hidden_1", ns: "sledge.alerts", background: true } 2015-10-27T20:08:02.358Z I INDEX [conn215819] build index done. scanned 0 total records. 0 secs 2015-10-27T20:08:04.003Z I - [conn215844] Index Build (background): 155000/165692 93% 2015-10-27T20:08:04.490Z I INDEX [conn215844] build index done. scanned 165692 total records. 11 secs 2015-10-27T20:08:07.523Z I INDEX [conn215819] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, data.endpoint_id: 1, alert_type: 1 }, name: "customer_id_1_data.endpoint_id_1_alert_type_1", ns: "sledge.alerts", background: true } 2015-10-27T20:08:07.526Z I INDEX [conn215849] build index on: sledge.message_schedules properties: { v: 1, key: { scheduled_at: 1 }, name: "scheduled_at_1", ns: "sledge.message_schedules", background: true } 2015-10-27T20:08:07.535Z I INDEX [conn215819] build index done. scanned 0 total records. 0 secs 2015-10-27T20:08:07.535Z I INDEX [conn215849] build index done. scanned 1 total records. 0 secs 2015-10-27T20:08:08.333Z I INDEX [conn215819] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, data.threat_id: 1, alert_type: 1 }, name: "customer_id_1_data.threat_id_1_alert_type_1", ns: "sledge.alerts", background: true } 2015-10-27T20:08:08.334Z I INDEX [conn215819] build index done. scanned 0 total records. 0 secs 2015-10-27T20:08:08.489Z I INDEX [conn215819] build index on: sledge.alerts properties: { v: 1, key: { endpoint_id: 1 }, name: "endpoint_id_1", ns: "sledge.alerts", background: true, sparse: true } 2015-10-27T20:08:08.489Z I INDEX [conn215819] build index done. scanned 0 total records. 0 secs 2015-10-27T20:08:08.945Z I INDEX [conn215869] build index on: sledge.endpoint_devices properties: { v: 1, key: { customer_id: 1, detected_at: 1 }, name: "customer_id_1_detected_at_1", ns: "sledge.endpoint_devices", background: true } 2015-10-27T20:08:11.008Z I - [conn215869] Index Build (background): 18500/20407 90% 2015-10-27T20:08:11.165Z I INDEX [conn215869] build index done. scanned 20407 total records. 2 secs 2015-10-27T20:08:11.273Z I INDEX [conn215869] build index on: sledge.endpoint_devices properties: { v: 1, key: { endpoint_id: 1, device_id: 1, instance_id: 1 }, name: "endpoint_id_1_device_id_1_instance_id_1", ns: "sledge.endpoint_devices", background: true } 2015-10-27T20:08:12.573Z I INDEX [conn215869] build index done. scanned 20408 total records. 1 secs 2015-10-27T20:08:12.582Z I INDEX [conn215869] build index on: sledge.endpoint_devices properties: { v: 1, key: { endpoint_id: 1, device_id: 1, device_type: 1, instance_id: 1, description: 1, detected_at: 1 }, name: "endpoint_id_1_device_id_1_device_type_1_instance_id_1_description_1_detected_at_1", ns: "sledge.endpoint_devices", background: true } 2015-10-27T20:08:13.320Z I INDEX [conn215943] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.report_bases", background: true } 2015-10-27T20:08:14.330Z I INDEX [conn215869] build index done. scanned 20409 total records. 1 secs 2015-10-27T20:08:15.787Z I INDEX [conn215943] build index done. scanned 48403 total records. 2 secs 2015-10-27T20:08:15.871Z I INDEX [conn215869] build index on: sledge.endpoint_devices properties: { v: 1, key: { customer_id: 1, device_type: 1, detected_at: 1 }, name: "customer_id_1_device_type_1_detected_at_1", ns: "sledge.endpoint_devices", background: true } 2015-10-27T20:08:15.874Z I INDEX [conn215943] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1 }, name: "customer_id_1__type_1", ns: "sledge.report_bases", background: true } 2015-10-27T20:08:16.904Z I INDEX [conn215869] build index done. scanned 20411 total records. 1 secs 2015-10-27T20:08:17.460Z I INDEX [conn215943] build index done. scanned 48410 total records. 1 secs 2015-10-27T20:08:17.473Z I INDEX [conn215943] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, tm: 1 }, name: "customer_id_1__type_1_tm_1", ns: "sledge.report_bases", background: true } 2015-10-27T20:08:17.609Z I INDEX [conn216010] build index on: sledge.report_bases properties: { v: 1, key: { ham_idx_upd: -1 }, name: "ham_idx_upd_-1", ns: "sledge.report_bases", background: true, sparse: true } 2015-10-27T20:08:17.708Z I INDEX [conn216010] build index done. scanned 48410 total records. 0 secs 2015-10-27T20:08:17.902Z I INDEX [conn216008] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, dom: 1 }, name: "customer_id_1_dom_1", ns: "sledge.report_bases", background: true } 2015-10-27T20:08:19.615Z I INDEX [conn215943] build index done. scanned 48413 total records. 2 secs 2015-10-27T20:08:20.003Z I - [conn216008] Index Build (background): 35100/48413 72% 2015-10-27T20:08:20.640Z I INDEX [conn215943] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, cat: 1 }, name: "customer_id_1_cat_1", ns: "sledge.report_bases", background: true } 2015-10-27T20:08:20.739Z I INDEX [conn216008] build index done. scanned 48416 total records. 2 secs 2015-10-27T20:08:20.819Z I INDEX [conn216008] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, rsn: 1 }, name: "customer_id_1_rsn_1", ns: "sledge.report_bases", background: true } 2015-10-27T20:08:23.003Z I - [conn215943] Index Build (background): 32400/48425 66% 2015-10-27T20:08:23.004Z I - [conn216008] Index Build (background): 34400/48425 71% 2015-10-27T20:08:23.518Z I INDEX [conn216008] build index done. scanned 48425 total records. 2 secs 2015-10-27T20:08:23.543Z I INDEX [conn216008] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, hits.type: 1, dom: 1 }, name: "customer_id_1_hits.type_1_dom_1", ns: "sledge.report_bases", background: true } 2015-10-27T20:08:23.590Z I INDEX [conn215943] build index done. scanned 48425 total records. 2 secs 2015-10-27T20:08:23.636Z I INDEX [conn215981] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, dom: 1, cat: 1, rsn: 1 }, name: "customer_id_1__type_1_dom_1_cat_1_rsn_1", ns: "sledge.report_bases", background: true } 2015-10-27T20:08:26.007Z I - [conn216008] Index Build (background): 30500/48428 62% 2015-10-27T20:08:26.009Z I - [conn215981] Index Build (background): 31500/48428 65% 2015-10-27T20:08:27.016Z I INDEX [conn216008] build index done. scanned 48429 total records. 3 secs 2015-10-27T20:08:27.060Z I INDEX [conn216006] build index on: sledge.endpoint_threats properties: { v: 1, key: { endpoint_id: 1, threat_id: 1 }, name: "endpoint_id_1_threat_id_1", ns: "sledge.endpoint_threats", background: true } 2015-10-27T20:08:27.236Z I INDEX [conn215981] build index done. scanned 48429 total records. 3 secs 2015-10-27T20:08:27.619Z I INDEX [conn216006] build index done. scanned 10291 total records. 0 secs 2015-10-27T20:08:27.624Z I INDEX [conn216006] build index on: sledge.endpoint_threats properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.endpoint_threats", background: true } 2015-10-27T20:08:27.771Z I INDEX [conn215954] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, name: 1, group: 1 }, name: "customer_id_1_name_1_group_1", ns: "sledge.report_bases", background: true } 2015-10-27T20:08:27.969Z I INDEX [conn216006] build index done. scanned 10291 total records. 0 secs 2015-10-27T20:08:28.202Z I INDEX [conn216018] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, hits.type: 1, name: 1, group: 1 }, name: "customer_id_1_hits.type_1_name_1_group_1", ns: "sledge.report_bases", background: true } 2015-10-27T20:08:30.002Z I - [conn215954] Index Build (background): 46200/48436 95% 2015-10-27T20:08:30.050Z I INDEX [conn215954] build index done. scanned 48436 total records. 2 secs 2015-10-27T20:08:30.067Z I INDEX [conn215954] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, name: 1 }, name: "customer_id_1__type_1_name_1", ns: "sledge.report_bases", background: true } 2015-10-27T20:08:30.869Z I INDEX [conn216018] build index done. scanned 48437 total records. 2 secs 2015-10-27T20:08:31.131Z I INDEX [conn216020] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, usr: 1 }, name: "customer_id_1_usr_1", ns: "sledge.report_bases", background: true } 2015-10-27T20:08:32.471Z I - [conn207473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:08:32.481Z I CONTROL [conn207473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:08:32.530Z I INDEX [conn215954] build index done. scanned 48444 total records. 2 secs 2015-10-27T20:08:32.980Z I INDEX [conn215924] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, loc: 1 }, name: "customer_id_1_loc_1", ns: "sledge.report_bases", background: true } 2015-10-27T20:08:33.237Z I INDEX [conn216020] build index done. scanned 48446 total records. 2 secs 2015-10-27T20:08:33.278Z I INDEX [conn216020] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, hits.type: 1, usr: 1 }, name: "customer_id_1_hits.type_1_usr_1", ns: "sledge.report_bases", background: true } 2015-10-27T20:08:34.845Z I INDEX [conn215924] build index done. scanned 48449 total records. 1 secs 2015-10-27T20:08:34.863Z I INDEX [conn215924] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, usr: 1, loc: 1 }, name: "customer_id_1__type_1_usr_1_loc_1", ns: "sledge.report_bases", background: true } 2015-10-27T20:08:34.957Z I INDEX [conn216020] build index done. scanned 48449 total records. 1 secs 2015-10-27T20:08:36.969Z I INDEX [conn215924] build index done. scanned 48451 total records. 2 secs 2015-10-27T20:08:39.861Z I - [conn215703] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:08:39.875Z I CONTROL [conn215703] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:08:51.608Z I - [conn205708] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:08:51.619Z I CONTROL [conn205708] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:09:12.878Z I - [conn205948] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:09:12.890Z I CONTROL [conn205948] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:09:43.240Z I - [conn205949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:09:43.250Z I CONTROL [conn205949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:10:05.440Z I - [conn205818] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:10:05.450Z I CONTROL [conn205818] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:10:23.935Z I - [conn205752] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:10:23.945Z I CONTROL [conn205752] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:11:18.344Z I - [conn215953] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:11:18.354Z I CONTROL [conn215953] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:11:38.634Z I - [conn205457] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:11:38.645Z I CONTROL [conn205457] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:12:42.448Z I - [conn205486] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:12:42.458Z I CONTROL [conn205486] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:13:21.036Z I - [conn205453] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:13:21.046Z I CONTROL [conn205453] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:14:00.965Z I - [conn215876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:14:00.978Z I CONTROL [conn215876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:14:19.823Z I - [conn215971] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:14:19.837Z I CONTROL [conn215971] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:14:39.758Z I NETWORK [conn215751] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:50958] 2015-10-27T20:14:40.352Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:14:40.362Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:15:02.021Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:15:02.032Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:15:19.525Z I - [conn205561] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:15:19.537Z I CONTROL [conn205561] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:16:00.099Z I - [conn205848] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:16:00.119Z I CONTROL [conn205848] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:16:21.205Z I - [conn215852] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:16:21.216Z I CONTROL [conn215852] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:16:41.617Z I - [conn212227] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:16:41.627Z I CONTROL [conn212227] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:17:03.571Z I - [conn170487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:17:03.581Z I CONTROL [conn170487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:17:40.312Z I - [conn205451] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:17:40.321Z I CONTROL [conn205451] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:17:59.930Z I - [conn205828] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:17:59.945Z I CONTROL [conn205828] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:18:21.814Z I - [conn215147] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:18:21.828Z I CONTROL [conn215147] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:18:39.345Z I - [conn177379] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:18:39.356Z I CONTROL [conn177379] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:18:59.174Z I - [conn205930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:18:59.185Z I CONTROL [conn205930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:19:15.341Z I - [conn206773] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:19:15.352Z I CONTROL [conn206773] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:19:36.471Z I - [conn216143] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:19:36.482Z I CONTROL [conn216143] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:19:54.684Z I - [conn205931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:19:54.696Z I CONTROL [conn205931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:20:14.337Z I - [conn215934] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:20:14.348Z I CONTROL [conn215934] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:20:37.739Z I - [conn183104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:20:37.749Z I CONTROL [conn183104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:20:59.751Z I - [conn205943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:20:59.761Z I CONTROL [conn205943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:21:14.381Z I - [conn206806] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:21:14.391Z I CONTROL [conn206806] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:21:35.791Z I - [conn206304] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:21:35.804Z I CONTROL [conn206304] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:21:53.357Z I - [conn205721] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:21:53.372Z I CONTROL [conn205721] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:22:13.346Z I - [conn216060] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:22:13.356Z I CONTROL [conn216060] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:22:33.371Z I - [conn205570] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:22:33.381Z I CONTROL [conn205570] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:22:53.365Z I - [conn216051] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:22:53.376Z I CONTROL [conn216051] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:23:13.383Z I - [conn207406] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:23:13.394Z I CONTROL [conn207406] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:23:33.356Z I - [conn205478] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:23:33.366Z I CONTROL [conn205478] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:23:53.356Z I - [conn207406] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:23:53.372Z I CONTROL [conn207406] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:24:13.355Z I - [conn205843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:24:13.366Z I CONTROL [conn205843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:24:33.344Z I - [conn205736] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:24:33.355Z I CONTROL [conn205736] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:24:53.358Z I - [conn216127] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:24:53.369Z I CONTROL [conn216127] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:25:13.355Z I - [conn216139] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:25:13.365Z I CONTROL [conn216139] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:25:33.356Z I - [conn205706] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:25:33.366Z I CONTROL [conn205706] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:25:53.380Z I - [conn205796] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:25:53.391Z I CONTROL [conn205796] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:26:13.360Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:26:13.370Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:26:33.348Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:26:33.357Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:26:53.360Z I - [conn205831] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:26:53.372Z I CONTROL [conn205831] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:27:13.359Z I - [conn215944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:27:13.368Z I CONTROL [conn215944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:27:33.345Z I - [conn216118] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:27:33.356Z I CONTROL [conn216118] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:27:53.356Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:27:53.366Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:28:13.356Z I - [conn170472] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:28:13.365Z I CONTROL [conn170472] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:28:33.354Z I - [conn207685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:28:33.364Z I CONTROL [conn207685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:28:53.364Z I - [conn216068] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:28:53.374Z I CONTROL [conn216068] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:29:13.352Z I - [conn205618] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:29:13.362Z I CONTROL [conn205618] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:29:33.366Z I - [conn205697] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:29:33.377Z I CONTROL [conn205697] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:29:53.357Z I - [conn205858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:29:53.368Z I CONTROL [conn205858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:30:13.352Z I - [conn215951] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:30:13.362Z I CONTROL [conn215951] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:30:33.440Z I - [conn183114] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:30:33.450Z I CONTROL [conn183114] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:30:53.342Z I - [conn215925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:30:53.352Z I CONTROL [conn215925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:31:13.372Z I - [conn206771] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:31:13.382Z I CONTROL [conn206771] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:31:33.358Z I - [conn206774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:31:33.368Z I CONTROL [conn206774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:31:53.363Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:31:53.372Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:39:49.412Z I - [conn216106] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:39:49.422Z I CONTROL [conn216106] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:40:28.808Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:40:28.819Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:40:47.706Z I - [conn215880] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:40:47.715Z I CONTROL [conn215880] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:41:07.574Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:41:07.583Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:41:25.772Z I - [conn205453] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:41:25.781Z I CONTROL [conn205453] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:41:43.225Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:41:43.234Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:42:03.049Z I - [conn215930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:42:03.060Z I CONTROL [conn215930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:42:23.407Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:42:23.416Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:42:41.539Z I - [conn205899] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:42:41.549Z I CONTROL [conn205899] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:42:59.292Z I - [conn205848] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:42:59.302Z I CONTROL [conn205848] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:43:19.279Z I - [conn215929] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:43:19.288Z I CONTROL [conn215929] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:43:39.289Z I - [conn215143] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:43:39.299Z I CONTROL [conn215143] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:43:59.295Z I - [conn212225] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:43:59.305Z I CONTROL [conn212225] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:44:19.296Z I - [conn205813] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:44:19.305Z I CONTROL [conn205813] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:44:39.292Z I - [conn206775] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:44:39.302Z I CONTROL [conn206775] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:44:59.287Z I - [conn216141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:44:59.297Z I CONTROL [conn216141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:45:19.295Z I - [conn170471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:45:19.305Z I CONTROL [conn170471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:45:39.272Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:45:39.281Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:45:59.293Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:45:59.302Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:46:19.284Z I - [conn207408] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:46:19.293Z I CONTROL [conn207408] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:46:39.280Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:46:39.290Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:46:59.278Z I - [conn205876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:46:59.287Z I CONTROL [conn205876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:47:19.273Z I - [conn215139] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:47:19.283Z I CONTROL [conn215139] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:47:39.296Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:47:39.305Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:47:59.274Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:47:59.283Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:48:19.270Z I - [conn171685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:48:19.279Z I CONTROL [conn171685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:48:39.275Z I - [conn205500] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:48:39.285Z I CONTROL [conn205500] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:48:59.289Z I - [conn215849] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:48:59.298Z I CONTROL [conn215849] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:49:19.270Z I - [conn205526] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:49:19.279Z I CONTROL [conn205526] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:49:39.268Z I - [conn205858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:49:39.277Z I CONTROL [conn205858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:49:59.281Z I - [conn216081] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:49:59.291Z I CONTROL [conn216081] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:50:19.284Z I - [conn215965] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:50:19.294Z I CONTROL [conn215965] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:50:39.290Z I - [conn206813] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:50:39.300Z I CONTROL [conn206813] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:50:59.280Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:50:59.290Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:51:19.270Z I - [conn215958] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:51:19.280Z I CONTROL [conn215958] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:51:39.287Z I - [conn215849] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:51:39.296Z I CONTROL [conn215849] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:51:59.279Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:51:59.289Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:56:49.666Z I - [conn215914] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:56:49.676Z I CONTROL [conn215914] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:57:14.633Z I - [conn216099] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:57:14.643Z I CONTROL [conn216099] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:57:30.795Z I - [conn215914] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:57:30.804Z I CONTROL [conn215914] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:57:50.139Z I - [conn216065] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:57:50.149Z I CONTROL [conn216065] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:58:05.496Z I - [conn216135] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:58:05.506Z I CONTROL [conn216135] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:58:25.456Z I - [conn205948] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:58:25.466Z I CONTROL [conn205948] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:58:45.451Z I - [conn205690] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:58:45.461Z I CONTROL [conn205690] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:59:05.464Z I - [conn205948] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:59:05.474Z I CONTROL [conn205948] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:59:25.450Z I - [conn215961] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:59:25.460Z I CONTROL [conn215961] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:59:45.454Z I - [conn215971] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T20:59:45.464Z I CONTROL [conn215971] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T20:59:49.406Z I INDEX [conn215735] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T21:00:05.472Z I - [conn215916] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:00:05.482Z I CONTROL [conn215916] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:00:25.461Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:00:25.471Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:00:45.440Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:00:45.450Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:01:05.454Z I - [conn215874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:01:05.464Z I CONTROL [conn215874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:01:25.461Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:01:25.471Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:01:45.446Z I - [conn215856] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:01:45.456Z I CONTROL [conn215856] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:02:05.465Z I - [conn216107] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:02:05.474Z I CONTROL [conn216107] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:02:25.466Z I - [conn216107] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:02:25.476Z I CONTROL [conn216107] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:02:45.454Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:02:45.464Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:03:05.451Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:03:05.462Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:03:25.458Z I - [conn177404] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:03:25.467Z I CONTROL [conn177404] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:03:45.448Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:03:45.458Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:04:05.451Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:04:05.461Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:04:25.451Z I - [conn216116] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:04:25.461Z I CONTROL [conn216116] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:04:45.458Z I - [conn216062] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:04:45.469Z I CONTROL [conn216062] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:05:05.460Z I - [conn170898] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:05:05.470Z I CONTROL [conn170898] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:05:26.725Z I - [conn205644] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:05:26.734Z I CONTROL [conn205644] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:05:45.462Z I - [conn205808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:05:45.471Z I CONTROL [conn205808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:06:05.453Z I - [conn206318] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:06:05.463Z I CONTROL [conn206318] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:06:25.451Z I - [conn215971] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:06:25.461Z I CONTROL [conn215971] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:06:45.449Z I - [conn216153] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:06:45.460Z I CONTROL [conn216153] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:07:05.452Z I - [conn206318] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:07:05.463Z I CONTROL [conn206318] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:07:25.447Z I - [conn215875] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:07:25.457Z I CONTROL [conn215875] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:07:45.452Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:07:45.461Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:08:05.451Z I - [conn205633] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:08:05.461Z I CONTROL [conn205633] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:08:25.447Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:08:25.456Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:08:45.448Z I - [conn216091] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:08:45.459Z I CONTROL [conn216091] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:09:05.445Z I - [conn216125] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:09:05.456Z I CONTROL [conn216125] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:09:25.456Z I - [conn215921] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:09:25.465Z I CONTROL [conn215921] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:09:45.451Z I - [conn205822] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:09:45.461Z I CONTROL [conn205822] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:10:05.447Z I - [conn216145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:10:05.457Z I CONTROL [conn216145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:10:25.447Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:10:25.456Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:10:45.449Z I - [conn205828] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:10:45.460Z I CONTROL [conn205828] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:11:05.452Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:11:05.463Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:11:25.447Z I - [conn205659] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:11:25.457Z I CONTROL [conn205659] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:11:45.457Z I - [conn207414] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:11:45.466Z I CONTROL [conn207414] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:12:05.451Z I - [conn205562] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:12:05.461Z I CONTROL [conn205562] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:12:25.455Z I - [conn205931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:12:25.464Z I CONTROL [conn205931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:12:45.459Z I - [conn215963] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:12:45.469Z I CONTROL [conn215963] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:13:05.453Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:13:05.463Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:13:25.454Z I - [conn215941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:13:25.464Z I CONTROL [conn215941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:13:45.448Z I - [conn205707] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:13:45.457Z I CONTROL [conn205707] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:14:05.455Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:14:05.464Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:14:25.454Z I - [conn205760] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:14:25.464Z I CONTROL [conn205760] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:14:45.448Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:14:45.457Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:15:05.455Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:15:05.465Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:15:25.443Z I - [conn205900] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:15:25.453Z I CONTROL [conn205900] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:15:45.442Z I - [conn216112] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:15:45.453Z I CONTROL [conn216112] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:16:05.453Z I - [conn216131] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:16:05.462Z I CONTROL [conn216131] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:16:25.455Z I - [conn215914] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:16:25.465Z I CONTROL [conn215914] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:16:45.459Z I - [conn216132] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:16:45.469Z I CONTROL [conn216132] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:17:05.442Z I - [conn205760] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:17:05.452Z I CONTROL [conn205760] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:17:25.443Z I - [conn205585] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:17:25.452Z I CONTROL [conn205585] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:17:45.455Z I - [conn205424] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:17:45.465Z I CONTROL [conn205424] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:18:05.443Z I - [conn212599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:18:05.452Z I CONTROL [conn212599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:18:25.444Z I - [conn205424] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:18:25.454Z I CONTROL [conn205424] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:18:45.433Z I - [conn216124] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:18:45.442Z I CONTROL [conn216124] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:19:05.442Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:19:05.452Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:19:25.451Z I - [conn177419] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:19:25.461Z I CONTROL [conn177419] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:19:45.434Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:19:45.443Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:20:05.441Z I - [conn207405] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:20:05.451Z I CONTROL [conn207405] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:20:25.449Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:20:25.459Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:20:45.438Z I - [conn216148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:20:45.449Z I CONTROL [conn216148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:21:05.441Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:21:05.451Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:21:25.442Z I - [conn205671] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:21:25.452Z I CONTROL [conn205671] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:21:45.438Z I - [conn216137] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:21:45.447Z I CONTROL [conn216137] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:22:05.441Z I - [conn206257] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:22:05.450Z I CONTROL [conn206257] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:22:25.442Z I - [conn205478] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:22:25.452Z I CONTROL [conn205478] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:22:45.444Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:22:45.454Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:23:05.452Z I - [conn205478] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:23:05.461Z I CONTROL [conn205478] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:23:25.439Z I - [conn215978] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:23:25.449Z I CONTROL [conn215978] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:23:45.437Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T21:23:45.447Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T21:59:50.172Z I INDEX [conn215461] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T22:35:31.548Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:35:31.557Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:35:51.555Z I - [conn205798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:35:51.565Z I CONTROL [conn205798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:36:11.554Z I - [conn206257] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:36:11.564Z I CONTROL [conn206257] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:36:31.545Z I - [conn215134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:36:31.555Z I CONTROL [conn215134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:36:51.551Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:36:51.561Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:37:11.551Z I - [conn215145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:37:11.561Z I CONTROL [conn215145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:37:31.548Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:37:31.558Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:37:51.549Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:37:51.559Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:38:11.551Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:38:11.560Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:38:31.553Z I - [conn215145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:38:31.563Z I CONTROL [conn215145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:38:51.548Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:38:51.557Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:39:11.547Z I - [conn215929] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:39:11.556Z I CONTROL [conn215929] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:39:31.553Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:39:31.562Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:39:51.549Z I - [conn215145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:39:51.558Z I CONTROL [conn215145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:40:11.541Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:40:11.551Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:40:31.552Z I - [conn216127] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:40:31.562Z I CONTROL [conn216127] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:40:51.550Z I - [conn215134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:40:51.559Z I CONTROL [conn215134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:41:11.549Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:41:11.559Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:41:31.549Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:41:31.559Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:41:51.552Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:41:51.562Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:42:11.538Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:42:11.548Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:42:31.544Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:42:31.554Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:42:51.548Z I - [conn183104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:42:51.558Z I CONTROL [conn183104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:43:11.547Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:43:11.557Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:43:31.544Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:43:31.553Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:43:51.550Z I - [conn215145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:43:51.559Z I CONTROL [conn215145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:44:11.547Z I - [conn205816] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:44:11.557Z I CONTROL [conn205816] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:44:31.547Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:44:31.556Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:44:51.551Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:44:51.560Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:45:11.556Z I - [conn216127] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:45:11.565Z I CONTROL [conn216127] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:45:31.545Z I - [conn205801] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:45:31.555Z I CONTROL [conn205801] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:45:51.548Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:45:51.558Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:46:11.536Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:46:11.545Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:46:31.545Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:46:31.555Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:46:51.544Z I - [conn215929] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:46:51.554Z I CONTROL [conn215929] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:47:11.546Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:47:11.555Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:47:31.546Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:47:31.556Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:47:51.553Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:47:51.562Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:48:11.567Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:48:11.577Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:48:31.545Z I - [conn215979] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:48:31.554Z I CONTROL [conn215979] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:48:51.559Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:48:51.569Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:49:11.550Z I - [conn205816] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:49:11.560Z I CONTROL [conn205816] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:49:31.546Z I - [conn215134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:49:31.556Z I CONTROL [conn215134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:49:51.546Z I - [conn216127] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:49:51.555Z I CONTROL [conn216127] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:50:11.548Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:50:11.558Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:50:31.547Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:50:31.556Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:50:51.537Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:50:51.547Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:51:11.541Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:51:11.551Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:51:31.548Z I - [conn215134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:51:31.558Z I CONTROL [conn215134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:51:51.539Z I - [conn216127] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:51:51.549Z I CONTROL [conn216127] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:52:11.544Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:52:11.554Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:52:31.541Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:52:31.550Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:52:51.541Z I - [conn215134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:52:51.550Z I CONTROL [conn215134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:53:11.541Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:53:11.551Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:53:31.537Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:53:31.546Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:53:51.539Z I - [conn215966] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:53:51.549Z I CONTROL [conn215966] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:54:11.540Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:54:11.550Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:54:31.544Z I - [conn215134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:54:31.554Z I CONTROL [conn215134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:54:51.541Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:54:51.550Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:55:11.531Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:55:11.540Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:55:31.539Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:55:31.549Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:55:51.538Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:55:51.548Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:56:11.542Z I - [conn205816] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:56:11.551Z I CONTROL [conn205816] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:56:31.559Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:56:31.569Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:56:51.539Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:56:51.548Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:57:11.542Z I - [conn216128] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:57:11.552Z I CONTROL [conn216128] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:57:31.537Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:57:31.547Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:57:51.541Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:57:51.550Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:58:11.541Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:58:11.550Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:58:31.530Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:58:31.540Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:58:51.539Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:58:51.548Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:59:11.538Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:59:11.548Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:59:31.537Z I - [conn216127] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:59:31.547Z I CONTROL [conn216127] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T22:59:49.554Z I INDEX [conn215568] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T22:59:51.540Z I - [conn215134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T22:59:51.549Z I CONTROL [conn215134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:00:11.537Z I - [conn215966] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:00:11.546Z I CONTROL [conn215966] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:00:31.547Z I - [conn206257] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:00:31.557Z I CONTROL [conn206257] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:00:51.536Z I - [conn205901] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:00:51.545Z I CONTROL [conn205901] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:01:11.540Z I - [conn215134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:01:11.549Z I CONTROL [conn215134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:01:31.528Z I - [conn215145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:01:31.537Z I CONTROL [conn215145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:01:51.535Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:01:51.545Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:02:11.527Z I - [conn216127] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:02:11.536Z I CONTROL [conn216127] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:02:31.538Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:02:31.548Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:02:51.545Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:02:51.555Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:03:11.534Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:03:11.544Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:03:31.533Z I - [conn206257] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:03:31.542Z I CONTROL [conn206257] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:03:51.533Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:03:51.542Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:04:11.537Z I - [conn206257] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:04:11.547Z I CONTROL [conn206257] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:04:31.548Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:04:31.557Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:04:51.538Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:04:51.547Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:05:11.539Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:05:11.549Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:05:31.540Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:05:31.550Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:05:51.535Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:05:51.545Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:06:11.538Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:06:11.547Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:06:31.531Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:06:31.541Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:06:51.528Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:06:51.538Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:07:11.540Z I - [conn215145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:07:11.549Z I CONTROL [conn215145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:07:31.534Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:07:31.544Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:07:51.532Z I - [conn216095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:07:51.542Z I CONTROL [conn216095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:08:11.544Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:08:11.553Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:08:31.538Z I - [conn205691] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:08:31.547Z I CONTROL [conn205691] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:08:51.537Z I - [conn205526] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:08:51.546Z I CONTROL [conn205526] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:09:11.536Z I - [conn205930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:09:11.546Z I CONTROL [conn205930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:09:31.536Z I - [conn205737] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:09:31.545Z I CONTROL [conn205737] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:09:51.537Z I - [conn216131] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:09:51.547Z I CONTROL [conn216131] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:10:11.536Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:10:11.545Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:10:31.539Z I - [conn215134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:10:31.549Z I CONTROL [conn215134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:10:51.538Z I - [conn205853] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:10:51.547Z I CONTROL [conn205853] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:11:11.537Z I - [conn205656] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:11:11.547Z I CONTROL [conn205656] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:11:31.534Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:11:31.543Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:11:51.531Z I - [conn216147] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:11:51.540Z I CONTROL [conn216147] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:12:11.537Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:12:11.547Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:12:31.533Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:12:31.543Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:12:51.573Z I - [conn206330] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:12:51.582Z I CONTROL [conn206330] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:13:11.531Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:13:11.540Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:13:31.531Z I - [conn205500] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:13:31.541Z I CONTROL [conn205500] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:13:51.532Z I - [conn205843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:13:51.542Z I CONTROL [conn205843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:14:11.530Z I - [conn216131] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:14:11.540Z I CONTROL [conn216131] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:14:31.536Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:14:31.546Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:14:51.539Z I - [conn212596] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:14:51.549Z I CONTROL [conn212596] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:15:11.534Z I - [conn215855] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:15:11.543Z I CONTROL [conn215855] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:15:31.535Z I - [conn215855] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:15:31.544Z I CONTROL [conn215855] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:15:51.535Z I - [conn205853] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:15:51.545Z I CONTROL [conn205853] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:16:11.536Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:16:11.545Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:16:31.537Z I - [conn216104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:16:31.547Z I CONTROL [conn216104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:16:51.542Z I - [conn215968] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:16:51.551Z I CONTROL [conn215968] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:17:11.534Z I - [conn215968] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:17:11.544Z I CONTROL [conn215968] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:17:31.531Z I - [conn205626] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:17:31.541Z I CONTROL [conn205626] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:17:51.531Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:17:51.541Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:18:11.541Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:18:11.551Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:18:31.533Z I - [conn216127] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:18:31.543Z I CONTROL [conn216127] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:18:51.533Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:18:51.543Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:19:11.533Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:19:11.543Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:19:31.537Z I - [conn205752] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:19:31.547Z I CONTROL [conn205752] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:19:51.528Z I - [conn216146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:19:51.537Z I CONTROL [conn216146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:20:11.531Z I - [conn216156] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:20:11.540Z I CONTROL [conn216156] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:20:31.538Z I - [conn170812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:20:31.548Z I CONTROL [conn170812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:20:51.534Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:20:51.543Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:21:11.531Z I - [conn216131] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:21:11.541Z I CONTROL [conn216131] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:21:31.534Z I - [conn183088] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:21:31.543Z I CONTROL [conn183088] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:21:51.521Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:21:51.530Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:22:11.539Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:22:11.549Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:22:31.530Z I - [conn205570] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:22:31.540Z I CONTROL [conn205570] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:22:51.538Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:22:51.548Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:23:11.532Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:23:11.541Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:23:31.530Z I - [conn206771] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:23:31.540Z I CONTROL [conn206771] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:23:51.532Z I - [conn206813] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:23:51.541Z I CONTROL [conn206813] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:24:11.528Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:24:11.537Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:24:31.538Z I - [conn215874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:24:31.547Z I CONTROL [conn215874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:24:51.530Z I - [conn205853] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:24:51.540Z I CONTROL [conn205853] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:25:11.531Z I - [conn205587] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:25:11.540Z I CONTROL [conn205587] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:25:31.529Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:25:31.539Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:25:51.533Z I - [conn205815] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:25:51.542Z I CONTROL [conn205815] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:26:11.531Z I - [conn216125] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:26:11.540Z I CONTROL [conn216125] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:26:31.521Z I - [conn205787] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:26:31.531Z I CONTROL [conn205787] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:26:51.534Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:26:51.544Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:27:11.528Z I - [conn183114] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:27:11.538Z I CONTROL [conn183114] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:27:31.524Z I - [conn215913] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:27:31.534Z I CONTROL [conn215913] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:27:51.525Z I - [conn216124] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:27:51.535Z I CONTROL [conn216124] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:28:11.533Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:28:11.542Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:28:31.529Z I - [conn205918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:28:31.538Z I CONTROL [conn205918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:28:51.533Z I - [conn216069] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:28:51.543Z I CONTROL [conn216069] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:29:11.525Z I - [conn215953] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:29:11.535Z I CONTROL [conn215953] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:29:31.523Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:29:31.533Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:29:51.525Z I - [conn215911] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:29:51.534Z I CONTROL [conn215911] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:30:11.527Z I - [conn215911] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:30:11.536Z I CONTROL [conn215911] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:30:31.529Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:30:31.538Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:30:51.526Z I - [conn205482] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:30:51.535Z I CONTROL [conn205482] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:31:11.526Z I - [conn183103] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:31:11.535Z I CONTROL [conn183103] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:31:31.528Z I - [conn205595] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:31:31.537Z I CONTROL [conn205595] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:31:51.529Z I - [conn206298] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:31:51.538Z I CONTROL [conn206298] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:32:11.523Z I - [conn212600] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:32:11.533Z I CONTROL [conn212600] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:32:31.522Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:32:31.531Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:32:51.522Z I - [conn215929] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:32:51.532Z I CONTROL [conn215929] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:33:11.527Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:33:11.536Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:33:31.529Z I - [conn183088] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:33:31.538Z I CONTROL [conn183088] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:33:51.516Z I - [conn216104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:33:51.525Z I CONTROL [conn216104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:34:11.523Z I - [conn205742] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:34:11.533Z I CONTROL [conn205742] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:34:31.526Z I - [conn205874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:34:31.536Z I CONTROL [conn205874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:34:51.529Z I - [conn206298] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:34:51.539Z I CONTROL [conn206298] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:35:11.532Z I - [conn206813] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:35:11.541Z I CONTROL [conn206813] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:35:31.528Z I - [conn212230] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:35:31.537Z I CONTROL [conn212230] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:35:51.523Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:35:51.532Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:36:11.526Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:36:11.536Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:36:31.522Z I - [conn216107] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:36:31.531Z I CONTROL [conn216107] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:36:51.521Z I - [conn205808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:36:51.531Z I CONTROL [conn205808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:37:11.513Z I - [conn183336] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:37:11.522Z I CONTROL [conn183336] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:37:31.527Z I - [conn205928] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:37:31.537Z I CONTROL [conn205928] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:37:51.523Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:37:51.533Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:38:11.540Z I - [conn205664] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:38:11.549Z I CONTROL [conn205664] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:38:31.528Z I - [conn215755] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:38:31.538Z I CONTROL [conn215755] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:38:51.525Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:38:51.535Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:39:11.521Z I - [conn215874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:39:11.531Z I CONTROL [conn215874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:39:31.522Z I - [conn216138] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:39:31.531Z I CONTROL [conn216138] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:39:51.525Z I - [conn205664] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:39:51.535Z I CONTROL [conn205664] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:40:11.513Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:40:11.523Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:40:31.520Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:40:31.530Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:40:51.522Z I - [conn215134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:40:51.532Z I CONTROL [conn215134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:41:11.613Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:41:11.623Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:41:31.522Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:41:31.531Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:41:51.521Z I - [conn205736] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:41:51.530Z I CONTROL [conn205736] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:42:11.521Z I - [conn215909] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:42:11.531Z I CONTROL [conn215909] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:42:31.518Z I - [conn216128] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:42:31.528Z I CONTROL [conn216128] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:42:51.523Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:42:51.533Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:43:11.522Z I - [conn205707] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:43:11.532Z I CONTROL [conn205707] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:43:31.521Z I - [conn205829] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:43:31.531Z I CONTROL [conn205829] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:43:51.519Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:43:51.529Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:44:11.522Z I - [conn205518] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:44:11.532Z I CONTROL [conn205518] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:44:31.516Z I - [conn216113] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:44:31.525Z I CONTROL [conn216113] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:44:51.522Z I - [conn205948] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:44:51.531Z I CONTROL [conn205948] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:45:11.529Z I - [conn205791] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:45:11.539Z I CONTROL [conn205791] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:45:31.519Z I - [conn216136] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:45:31.529Z I CONTROL [conn216136] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:45:51.521Z I - [conn212600] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:45:51.530Z I CONTROL [conn212600] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:46:11.520Z I - [conn205872] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:46:11.529Z I CONTROL [conn205872] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:46:31.522Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:46:31.532Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:46:51.520Z I - [conn206807] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:46:51.530Z I CONTROL [conn206807] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:47:11.519Z I - [conn205621] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:47:11.528Z I CONTROL [conn205621] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:47:31.516Z I - [conn215963] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:47:31.526Z I CONTROL [conn215963] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:47:51.520Z I - [conn205457] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:47:51.530Z I CONTROL [conn205457] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:48:11.517Z I - [conn216095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:48:11.526Z I CONTROL [conn216095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:48:31.516Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:48:31.526Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:48:51.522Z I - [conn205644] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:48:51.532Z I CONTROL [conn205644] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:49:11.514Z I - [conn215910] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:49:11.524Z I CONTROL [conn215910] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:49:31.516Z I - [conn205787] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:49:31.525Z I CONTROL [conn205787] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:49:51.517Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:49:51.526Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:50:11.515Z I - [conn216069] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:50:11.524Z I CONTROL [conn216069] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:50:31.518Z I - [conn205930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:50:31.528Z I CONTROL [conn205930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:50:51.509Z I - [conn205586] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:50:51.519Z I CONTROL [conn205586] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:51:11.517Z I - [conn206811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:51:11.527Z I CONTROL [conn206811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:51:31.514Z I - [conn205548] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:51:31.523Z I CONTROL [conn205548] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:51:51.518Z I - [conn215957] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:51:51.528Z I CONTROL [conn215957] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:52:11.516Z I - [conn215975] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:52:11.526Z I CONTROL [conn215975] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:52:31.514Z I - [conn216156] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:52:31.524Z I CONTROL [conn216156] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:52:51.524Z I - [conn205617] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:52:51.534Z I CONTROL [conn205617] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:53:11.516Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:53:11.525Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:53:26.253Z I - [conn205482] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:53:26.263Z I CONTROL [conn205482] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:53:31.521Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:53:31.531Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:53:51.516Z I - [conn216128] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:53:51.526Z I CONTROL [conn216128] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:54:11.519Z I - [conn205500] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:54:11.528Z I CONTROL [conn205500] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:54:31.517Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:54:31.527Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:54:51.514Z I - [conn206329] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:54:51.523Z I CONTROL [conn206329] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:55:11.517Z I - [conn206811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:55:11.527Z I CONTROL [conn206811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:55:31.516Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:55:31.526Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:55:51.504Z I - [conn205554] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:55:51.513Z I CONTROL [conn205554] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:56:11.510Z I - [conn215944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:56:11.520Z I CONTROL [conn215944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:56:31.512Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:56:31.522Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:56:51.518Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:56:51.527Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:57:11.513Z I - [conn216113] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:57:11.523Z I CONTROL [conn216113] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:57:31.516Z I - [conn216113] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:57:31.526Z I CONTROL [conn216113] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:57:51.515Z I - [conn205517] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:57:51.525Z I CONTROL [conn205517] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:58:11.517Z I - [conn206811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:58:11.527Z I CONTROL [conn206811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:58:31.512Z I - [conn205928] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:58:31.522Z I CONTROL [conn205928] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:58:51.516Z I - [conn205928] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:58:51.525Z I CONTROL [conn205928] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:59:11.512Z I - [conn215876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:59:11.522Z I CONTROL [conn215876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:59:31.509Z I - [conn216129] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:59:31.519Z I CONTROL [conn216129] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-27T23:59:50.158Z I INDEX [conn215461] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-27T23:59:51.506Z I - [conn215938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-27T23:59:51.516Z I CONTROL [conn215938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:00:11.508Z I - [conn215938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:00:11.519Z I CONTROL [conn215938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:00:31.512Z I - [conn205944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:00:31.521Z I CONTROL [conn205944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:00:51.515Z I - [conn178530] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:00:51.524Z I CONTROL [conn178530] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:01:11.513Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:01:11.523Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:01:31.525Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:01:31.535Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:01:51.509Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:01:51.519Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:02:11.513Z I - [conn205945] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:02:11.522Z I CONTROL [conn205945] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:02:31.511Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:02:31.521Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:02:51.511Z I - [conn215874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:02:51.521Z I CONTROL [conn215874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:03:11.510Z I - [conn205424] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:03:11.519Z I CONTROL [conn205424] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:03:31.502Z I - [conn216062] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:03:31.512Z I CONTROL [conn216062] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:03:51.508Z I - [conn177379] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:03:51.517Z I CONTROL [conn177379] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:04:11.505Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:04:11.515Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:04:31.509Z I - [conn205858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:04:31.519Z I CONTROL [conn205858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:04:51.507Z I - [conn216081] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:04:51.516Z I CONTROL [conn216081] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:05:11.514Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:05:11.524Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:05:31.511Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:05:31.521Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:05:51.509Z I - [conn215139] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:05:51.518Z I CONTROL [conn215139] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:06:11.514Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:06:11.523Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:06:31.504Z I - [conn215913] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:06:31.514Z I CONTROL [conn215913] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:06:51.504Z I - [conn216089] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:06:51.514Z I CONTROL [conn216089] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:07:11.508Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:07:11.518Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:07:31.509Z I - [conn212230] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:07:31.518Z I CONTROL [conn212230] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:07:51.510Z I - [conn216069] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:07:51.519Z I CONTROL [conn216069] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:08:11.509Z I - [conn206813] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:08:11.519Z I CONTROL [conn206813] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:08:31.506Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:08:31.515Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:08:51.505Z I - [conn216107] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:08:51.514Z I CONTROL [conn216107] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:09:11.506Z I - [conn205915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:09:11.515Z I CONTROL [conn205915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:09:31.509Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:09:31.518Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:09:51.508Z I - [conn215140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:09:51.518Z I CONTROL [conn215140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:09:54.553Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:09:54.563Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:10:01.721Z I CONTROL [signalProcessingThread] pid=16473 port=27017 64-bit host=ip-172-19-100-23 2015-10-28T00:10:01.721Z I CONTROL [signalProcessingThread] db version v3.0.5 2015-10-28T00:10:01.721Z I CONTROL [signalProcessingThread] git version: 8bc4ae20708dbb493cb09338d9e7be6698e4a3a3 2015-10-28T00:10:01.721Z I CONTROL [signalProcessingThread] build info: Linux ip-10-230-137-3 3.4.43-43.43.amzn1.x86_64 #1 SMP Mon May 6 18:04:41 UTC 2013 x86_64 BOOST_LIB_VERSION=1_49 2015-10-28T00:10:01.721Z I CONTROL [signalProcessingThread] allocator: tcmalloc 2015-10-28T00:10:01.721Z I CONTROL [signalProcessingThread] options: { config: "/etc/mongod.conf", net: { port: 27017, ssl: { PEMKeyFile: "/mongodata/config/mongodb.pem", PEMKeyPassword: "", mode: "preferSSL" } }, operationProfiling: { mode: "off", slowOpThresholdMs: 100 }, processManagement: { fork: true, pidFilePath: "/var/run/mongodb/mongod.pid" }, replication: { oplogSizeMB: 20480, replSetName: "upe_dccb" }, security: { clusterAuthMode: "keyFile", keyFile: "/mongodata/config/keyfile" }, storage: { dbPath: "/mongodata/db", directoryPerDB: true, engine: "wiredTiger", wiredTiger: { collectionConfig: { blockCompressor: "snappy" }, engineConfig: { directoryForIndexes: true, journalCompressor: "snappy", statisticsLogDelaySecs: 0 }, indexConfig: { prefixCompression: true } } }, systemLog: { destination: "file", logAppend: true, logRotate: "reopen", path: "/data/log/mongodb/mongodb.log", timeStampFormat: "iso8601-utc", traceAllExceptions: false, verbosity: 0 } } 2015-10-28T00:10:11.507Z I - [conn205743] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:10:11.516Z I CONTROL [conn205743] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:10:31.510Z I - [conn205526] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:10:31.519Z I CONTROL [conn205526] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:10:51.506Z I - [conn216138] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:10:51.516Z I CONTROL [conn216138] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:11:11.501Z I - [conn216138] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:11:11.511Z I CONTROL [conn216138] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:11:31.517Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:11:31.527Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:11:51.507Z I - [conn206329] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:11:51.517Z I CONTROL [conn206329] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:12:11.504Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:12:11.513Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:12:31.519Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:12:31.529Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:12:51.507Z I - [conn215943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:12:51.516Z I CONTROL [conn215943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:13:11.507Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:13:11.517Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:13:31.504Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:13:31.514Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:13:51.949Z I - [conn205615] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:13:51.959Z I CONTROL [conn205615] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:14:11.506Z I - [conn216132] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:14:11.516Z I CONTROL [conn216132] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:14:31.507Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:14:31.516Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:14:51.502Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:14:51.512Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:15:11.509Z I - [conn215910] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:15:11.518Z I CONTROL [conn215910] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:15:31.501Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:15:31.511Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:15:51.504Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:15:51.514Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:16:11.498Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:16:11.508Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:16:31.500Z I - [conn216104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:16:31.509Z I CONTROL [conn216104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:16:51.500Z I - [conn215943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:16:51.510Z I CONTROL [conn215943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:17:11.513Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:17:11.523Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:17:31.503Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:17:31.513Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:17:51.511Z I - [conn216089] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:17:51.520Z I CONTROL [conn216089] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:18:11.507Z I - [conn215144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:18:11.517Z I CONTROL [conn215144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:18:31.502Z I - [conn205638] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:18:31.511Z I CONTROL [conn205638] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:18:51.502Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:18:51.512Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:19:11.502Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:19:11.512Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:19:31.505Z I - [conn205949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:19:31.516Z I CONTROL [conn205949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:19:51.492Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:19:51.502Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:20:11.496Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:20:11.505Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:20:31.515Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:20:31.525Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:20:51.493Z I - [conn205918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:20:51.502Z I CONTROL [conn205918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:21:11.504Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:21:11.514Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:21:31.500Z I - [conn205917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:21:31.510Z I CONTROL [conn205917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:21:51.503Z I - [conn205587] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:21:51.513Z I CONTROL [conn205587] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:22:11.499Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:22:11.508Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:22:31.501Z I - [conn206319] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:22:31.510Z I CONTROL [conn206319] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:22:51.494Z I - [conn205783] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:22:51.503Z I CONTROL [conn205783] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:23:11.500Z I - [conn183171] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:23:11.510Z I CONTROL [conn183171] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:23:31.501Z I - [conn215960] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:23:31.511Z I CONTROL [conn215960] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:23:51.502Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:23:51.511Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:24:11.499Z I - [conn183171] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:24:11.509Z I CONTROL [conn183171] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:24:31.508Z I - [conn216124] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:24:31.517Z I CONTROL [conn216124] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:24:51.499Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:24:51.509Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:25:11.498Z I - [conn216153] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:25:11.508Z I CONTROL [conn216153] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:25:31.500Z I - [conn215957] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:25:31.509Z I CONTROL [conn215957] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:25:51.500Z I - [conn205473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:25:51.510Z I CONTROL [conn205473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:26:11.503Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:26:11.512Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:26:31.493Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:26:31.503Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:26:51.511Z I - [conn177467] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:26:51.521Z I CONTROL [conn177467] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:27:11.537Z I - [conn177467] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:27:11.547Z I CONTROL [conn177467] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:27:31.498Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:27:31.508Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:27:51.498Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:27:51.508Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:28:11.489Z I - [conn216119] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:28:11.499Z I CONTROL [conn216119] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:28:31.500Z I - [conn206328] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:28:31.510Z I CONTROL [conn206328] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:28:51.501Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:28:51.511Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:29:11.488Z I - [conn207412] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:29:11.498Z I CONTROL [conn207412] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:29:31.494Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:29:31.504Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:29:51.489Z I - [conn205743] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:29:51.499Z I CONTROL [conn205743] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:30:11.492Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:30:11.501Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:30:31.499Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:30:31.509Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:30:51.487Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:30:51.496Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:31:11.487Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:31:11.497Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:31:31.495Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:31:31.505Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:31:51.499Z I - [conn205858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:31:51.509Z I CONTROL [conn205858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:32:11.499Z I - [conn170472] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:32:11.509Z I CONTROL [conn170472] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:32:31.491Z I - [conn205457] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:32:31.500Z I CONTROL [conn205457] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:32:51.497Z I - [conn205518] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:32:51.507Z I CONTROL [conn205518] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:33:11.491Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:33:11.500Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:33:33.647Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:33:33.656Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:33:51.490Z I - [conn206318] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:33:51.500Z I CONTROL [conn206318] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:34:11.493Z I - [conn215960] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:34:11.502Z I CONTROL [conn215960] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:34:31.498Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:34:31.507Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:34:51.485Z I - [conn215876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:34:51.495Z I CONTROL [conn215876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:35:11.481Z I - [conn215927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:35:11.491Z I CONTROL [conn215927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:35:31.490Z I - [conn216102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:35:31.499Z I CONTROL [conn216102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:35:51.486Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:35:51.495Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:36:11.505Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:36:11.515Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:36:31.487Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:36:31.496Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:36:51.492Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:36:51.501Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:37:11.490Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:37:11.500Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:37:31.493Z I - [conn205518] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:37:31.503Z I CONTROL [conn205518] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:37:51.484Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:37:51.494Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:38:11.494Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:38:11.503Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:38:31.492Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:38:31.501Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:38:51.493Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:38:51.503Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:39:11.487Z I - [conn205572] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:39:11.497Z I CONTROL [conn205572] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:39:31.489Z I - [conn215939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:39:31.498Z I CONTROL [conn215939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:39:51.488Z I - [conn216128] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:39:51.497Z I CONTROL [conn216128] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:40:11.498Z I - [conn178530] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:40:11.508Z I CONTROL [conn178530] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:40:31.483Z I - [conn215966] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:40:31.493Z I CONTROL [conn215966] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:40:51.491Z I - [conn216136] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:40:51.501Z I CONTROL [conn216136] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:41:11.478Z I - [conn215910] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:41:11.488Z I CONTROL [conn215910] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:41:31.490Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:41:31.500Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:41:51.479Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:41:51.489Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:42:11.495Z I - [conn215966] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:42:11.504Z I CONTROL [conn215966] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:42:31.489Z I - [conn205752] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:42:31.499Z I CONTROL [conn205752] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:42:51.489Z I - [conn207413] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:42:51.499Z I CONTROL [conn207413] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:43:11.493Z I - [conn205633] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:43:11.503Z I CONTROL [conn205633] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:43:31.490Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:43:31.499Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:43:51.480Z I - [conn205948] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:43:51.489Z I CONTROL [conn205948] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:44:11.495Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:44:11.504Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:44:31.493Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:44:31.503Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:44:51.492Z I - [conn205949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:44:51.502Z I CONTROL [conn205949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:45:11.491Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:45:11.501Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:45:31.494Z I - [conn205500] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:45:31.503Z I CONTROL [conn205500] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:45:51.478Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:45:51.487Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:46:11.492Z I - [conn216069] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:46:11.502Z I CONTROL [conn216069] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:46:31.481Z I - [conn215983] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:46:31.490Z I CONTROL [conn215983] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:46:51.488Z I - [conn207413] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:46:51.497Z I CONTROL [conn207413] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:47:11.489Z I - [conn215947] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:47:11.498Z I CONTROL [conn215947] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:47:31.487Z I - [conn207413] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:47:31.496Z I CONTROL [conn207413] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:47:51.500Z I - [conn170812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:47:51.509Z I CONTROL [conn170812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:48:11.487Z I - [conn205621] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:48:11.497Z I CONTROL [conn205621] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:48:31.493Z I - [conn205876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:48:31.503Z I CONTROL [conn205876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:48:51.488Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:48:51.498Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:49:11.498Z I - [conn205798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:49:11.507Z I CONTROL [conn205798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:49:31.481Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:49:31.491Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:49:51.488Z I - [conn206330] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:49:51.498Z I CONTROL [conn206330] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:50:11.487Z I - [conn205798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:50:11.497Z I CONTROL [conn205798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:50:31.487Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:50:31.496Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:50:51.490Z I - [conn216138] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:50:51.499Z I CONTROL [conn216138] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:51:11.487Z I - [conn205863] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:51:11.497Z I CONTROL [conn205863] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:51:31.480Z I - [conn215913] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:51:31.489Z I CONTROL [conn215913] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:51:51.477Z I - [conn205798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:51:51.487Z I CONTROL [conn205798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:52:11.484Z I - [conn205882] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:52:11.494Z I CONTROL [conn205882] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:52:31.486Z I - [conn205882] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:52:31.496Z I CONTROL [conn205882] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:52:51.479Z I - [conn205457] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:52:51.489Z I CONTROL [conn205457] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:53:11.488Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:53:11.498Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:53:31.489Z I - [conn205844] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:53:31.499Z I CONTROL [conn205844] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:53:51.481Z I - [conn216081] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:53:51.491Z I CONTROL [conn216081] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:54:11.484Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:54:11.493Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:54:31.484Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:54:31.493Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:54:51.488Z I - [conn216081] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:54:51.497Z I CONTROL [conn216081] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:55:11.485Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:55:11.495Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:55:31.479Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:55:31.489Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:55:51.484Z I - [conn206329] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:55:51.494Z I CONTROL [conn206329] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:56:11.485Z I - [conn205639] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:56:11.494Z I CONTROL [conn205639] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:56:31.481Z I - [conn206320] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:56:31.491Z I CONTROL [conn206320] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:56:51.482Z I - [conn205928] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:56:51.492Z I CONTROL [conn205928] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:57:11.482Z I - [conn206813] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:57:11.491Z I CONTROL [conn206813] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:57:31.483Z I - [conn177467] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:57:31.492Z I CONTROL [conn177467] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:57:51.487Z I - [conn205638] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:57:51.496Z I CONTROL [conn205638] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:58:11.470Z I - [conn216123] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:58:11.480Z I CONTROL [conn216123] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:58:31.482Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:58:31.492Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:58:51.484Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:58:51.494Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:59:11.482Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:59:11.492Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:59:31.489Z I - [conn215918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:59:31.499Z I CONTROL [conn215918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T00:59:49.177Z I INDEX [conn215461] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T00:59:51.480Z I - [conn215943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T00:59:51.490Z I CONTROL [conn215943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:00:11.482Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:00:11.491Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:00:31.487Z I - [conn177419] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:00:31.496Z I CONTROL [conn177419] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:00:51.484Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:00:51.494Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:01:11.479Z I - [conn205737] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:01:11.488Z I CONTROL [conn205737] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:01:31.479Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:01:31.488Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:01:51.478Z I - [conn215876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:01:51.488Z I CONTROL [conn215876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:02:11.477Z I - [conn215876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:02:11.487Z I CONTROL [conn215876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:02:31.471Z I - [conn212596] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:02:31.480Z I CONTROL [conn212596] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:02:51.477Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:02:51.486Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:03:11.480Z I - [conn205933] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:03:11.489Z I CONTROL [conn205933] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:03:31.466Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:03:31.476Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:03:51.480Z I - [conn206771] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:03:51.490Z I CONTROL [conn206771] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:04:11.478Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:04:11.488Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:04:31.478Z I - [conn205843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:04:31.487Z I CONTROL [conn205843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:04:51.476Z I - [conn177467] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:04:51.486Z I CONTROL [conn177467] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:05:11.478Z I - [conn215140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:05:11.488Z I CONTROL [conn215140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:05:31.480Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:05:31.490Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:05:51.467Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:05:51.477Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:06:11.477Z I - [conn215965] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:06:11.487Z I CONTROL [conn215965] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:06:31.469Z I - [conn205882] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:06:31.478Z I CONTROL [conn205882] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:06:51.484Z I - [conn205457] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:06:51.494Z I CONTROL [conn205457] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:07:11.468Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:07:11.478Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:07:31.465Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:07:31.475Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:07:51.478Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:07:51.488Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:08:11.465Z I - [conn215855] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:08:11.475Z I CONTROL [conn215855] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:08:31.485Z I - [conn215755] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:08:31.494Z I CONTROL [conn215755] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:08:51.472Z I - [conn205671] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:08:51.481Z I CONTROL [conn205671] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:09:11.473Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:09:11.482Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:09:31.468Z I - [conn205671] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:09:31.478Z I CONTROL [conn205671] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:09:51.473Z I - [conn205607] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:09:51.484Z I CONTROL [conn205607] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:10:11.476Z I - [conn216153] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:10:11.486Z I CONTROL [conn216153] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:10:31.476Z I - [conn216153] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:10:31.485Z I CONTROL [conn216153] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:10:51.475Z I - [conn205607] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:10:51.486Z I CONTROL [conn205607] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:11:11.477Z I - [conn205453] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:11:11.487Z I CONTROL [conn205453] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:11:31.471Z I - [conn216089] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:11:31.481Z I CONTROL [conn216089] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:11:51.477Z I - [conn206316] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:11:51.486Z I CONTROL [conn206316] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:12:11.475Z I - [conn177419] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:12:11.484Z I CONTROL [conn177419] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:12:31.475Z I - [conn205944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:12:31.484Z I CONTROL [conn205944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:12:51.476Z I - [conn206811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:12:51.486Z I CONTROL [conn206811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:13:11.462Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:13:11.472Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:13:31.464Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:13:31.474Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:13:51.484Z I - [conn216129] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:13:51.493Z I CONTROL [conn216129] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:14:11.484Z I - [conn215971] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:14:11.494Z I CONTROL [conn215971] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:14:31.475Z I - [conn205526] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:14:31.484Z I CONTROL [conn205526] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:14:51.472Z I - [conn205595] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:14:51.482Z I CONTROL [conn205595] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:15:11.471Z I - [conn216102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:15:11.481Z I CONTROL [conn216102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:15:31.478Z I - [conn215976] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:15:31.488Z I CONTROL [conn215976] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:15:51.474Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:15:51.484Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:16:11.475Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:16:11.484Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:16:31.477Z I - [conn215139] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:16:31.487Z I CONTROL [conn215139] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:16:51.470Z I - [conn183114] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:16:51.480Z I CONTROL [conn183114] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:17:11.474Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:17:11.484Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:17:31.468Z I - [conn206329] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:17:31.478Z I CONTROL [conn206329] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:17:51.473Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:17:51.482Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:18:11.467Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:18:11.476Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:18:31.476Z I - [conn215971] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:18:31.485Z I CONTROL [conn215971] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:18:51.483Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:18:51.492Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:19:11.469Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:19:11.478Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:19:31.463Z I - [conn205752] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:19:31.472Z I CONTROL [conn205752] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:19:51.465Z I - [conn216145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:19:51.475Z I CONTROL [conn216145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:20:11.493Z I - [conn215929] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:20:11.503Z I CONTROL [conn215929] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:20:31.462Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:20:31.472Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:20:51.474Z I - [conn215876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:20:51.483Z I CONTROL [conn215876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:21:11.472Z I - [conn206319] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:21:11.482Z I CONTROL [conn206319] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:21:31.470Z I - [conn216062] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:21:31.479Z I CONTROL [conn216062] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:21:51.472Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:21:51.481Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:22:11.481Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:22:11.490Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:22:31.481Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:22:31.490Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:22:51.459Z I - [conn205548] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:22:51.469Z I CONTROL [conn205548] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:23:11.474Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:23:11.484Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:23:31.468Z I - [conn207412] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:23:31.477Z I CONTROL [conn207412] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:23:51.475Z I - [conn216156] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:23:51.485Z I CONTROL [conn216156] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:23:58.089Z I - [conn205815] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:23:58.098Z I CONTROL [conn205815] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:24:11.468Z I - [conn205664] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:24:11.478Z I CONTROL [conn205664] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:24:31.466Z I - [conn205804] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:24:31.476Z I CONTROL [conn205804] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:24:51.482Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:24:51.491Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:25:11.474Z I - [conn206319] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:25:11.483Z I CONTROL [conn206319] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:25:31.473Z I - [conn205857] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:25:31.483Z I CONTROL [conn205857] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:25:51.462Z I - [conn215953] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:25:51.472Z I CONTROL [conn215953] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:26:11.470Z I - [conn205633] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:26:11.480Z I CONTROL [conn205633] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:26:31.472Z I - [conn215143] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:26:31.481Z I CONTROL [conn215143] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:26:51.467Z I - [conn205829] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:26:51.476Z I CONTROL [conn205829] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:27:11.458Z I - [conn205917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:27:11.468Z I CONTROL [conn205917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:27:31.478Z I - [conn205471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:27:31.488Z I CONTROL [conn205471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:27:51.461Z I - [conn205856] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:27:51.470Z I CONTROL [conn205856] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:28:11.457Z I - [conn183115] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:28:11.467Z I CONTROL [conn183115] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:28:31.467Z I - [conn183115] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:28:31.476Z I CONTROL [conn183115] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:28:51.469Z I - [conn212225] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:28:51.479Z I CONTROL [conn212225] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:29:11.459Z I - [conn216125] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:29:11.469Z I CONTROL [conn216125] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:29:31.467Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:29:31.477Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:29:51.462Z I - [conn215983] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:29:51.471Z I CONTROL [conn215983] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:30:11.465Z I - [conn170471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:30:11.475Z I CONTROL [conn170471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:30:31.464Z I - [conn205865] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:30:31.474Z I CONTROL [conn205865] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:30:51.465Z I - [conn216069] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:30:51.474Z I CONTROL [conn216069] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:31:11.481Z I - [conn205858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:31:11.491Z I CONTROL [conn205858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:31:31.469Z I - [conn207412] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:31:31.478Z I CONTROL [conn207412] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:31:51.467Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:31:51.477Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:32:11.466Z I - [conn205816] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:32:11.475Z I CONTROL [conn205816] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:32:31.466Z I - [conn205815] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:32:31.476Z I CONTROL [conn205815] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:32:51.466Z I - [conn205500] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:32:51.475Z I CONTROL [conn205500] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:33:11.461Z I - [conn216118] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:33:11.471Z I CONTROL [conn216118] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:33:31.459Z I - [conn216119] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:33:31.469Z I CONTROL [conn216119] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:33:51.464Z I - [conn205633] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:33:51.473Z I CONTROL [conn205633] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:34:11.465Z I - [conn205937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:34:11.475Z I CONTROL [conn205937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:34:31.461Z I - [conn205737] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:34:31.471Z I CONTROL [conn205737] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:34:51.450Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:34:51.460Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:35:11.457Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:35:11.466Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:35:31.476Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:35:31.485Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:35:51.464Z I - [conn183103] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:35:51.473Z I CONTROL [conn183103] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:36:11.472Z I - [conn207405] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:36:11.481Z I CONTROL [conn207405] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:36:31.460Z I - [conn215983] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:36:31.470Z I CONTROL [conn215983] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:36:51.460Z I - [conn205810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:36:51.469Z I CONTROL [conn205810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:37:11.464Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:37:11.474Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:37:31.471Z I - [conn205681] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:37:31.481Z I CONTROL [conn205681] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:37:51.462Z I - [conn205681] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:37:51.472Z I CONTROL [conn205681] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:38:11.453Z I - [conn205804] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:38:11.463Z I CONTROL [conn205804] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:38:18.435Z I - [conn215927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:38:18.445Z I CONTROL [conn215927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:38:31.455Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:38:31.465Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:38:51.463Z I - [conn205626] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:38:51.473Z I CONTROL [conn205626] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:39:11.459Z I - [conn205691] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:39:11.469Z I CONTROL [conn205691] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:39:31.460Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:39:31.469Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:39:32.318Z I - [conn205878] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:39:32.327Z I CONTROL [conn205878] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:39:51.459Z I - [conn216146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:39:51.469Z I CONTROL [conn216146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:40:11.471Z I - [conn205478] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:40:11.481Z I CONTROL [conn205478] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:40:31.463Z I - [conn206775] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:40:31.472Z I CONTROL [conn206775] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:40:51.460Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:40:51.469Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:41:11.462Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:41:11.471Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:41:31.461Z I - [conn170812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:41:31.471Z I CONTROL [conn170812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:41:51.463Z I - [conn207413] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:41:51.473Z I CONTROL [conn207413] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:42:11.459Z I - [conn215854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:42:11.469Z I CONTROL [conn215854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:42:31.462Z I - [conn206771] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:42:31.472Z I CONTROL [conn206771] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:42:43.136Z I - [conn205626] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:42:43.145Z I CONTROL [conn205626] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:42:51.465Z I - [conn205626] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:42:51.474Z I CONTROL [conn205626] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:43:11.469Z I - [conn205664] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:43:11.479Z I CONTROL [conn205664] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:43:31.459Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:43:31.469Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:43:51.460Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:43:51.469Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:44:11.458Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:44:11.467Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:44:31.460Z I - [conn215145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:44:31.470Z I CONTROL [conn215145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:44:51.455Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:44:51.464Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:45:11.449Z I - [conn212225] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:45:11.459Z I CONTROL [conn212225] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:45:31.454Z I - [conn216132] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:45:31.464Z I CONTROL [conn216132] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:45:51.449Z I - [conn205499] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:45:51.458Z I CONTROL [conn205499] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:46:11.458Z I - [conn205787] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:46:11.468Z I CONTROL [conn205787] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:46:31.457Z I - [conn205798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:46:31.467Z I CONTROL [conn205798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:46:51.459Z I - [conn215919] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:46:51.468Z I CONTROL [conn215919] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:47:11.455Z I - [conn205865] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:47:11.466Z I CONTROL [conn205865] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:47:13.218Z I - [conn205726] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:47:13.228Z I CONTROL [conn205726] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:47:31.450Z I - [conn216089] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:47:31.459Z I CONTROL [conn216089] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:47:51.465Z I - [conn215930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:47:51.475Z I CONTROL [conn215930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:48:11.452Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:48:11.462Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:48:31.446Z I - [conn216129] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:48:31.455Z I CONTROL [conn216129] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:48:51.463Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:48:51.472Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:49:11.458Z I - [conn206775] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:49:11.467Z I CONTROL [conn206775] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:49:31.454Z I - [conn205707] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:49:31.464Z I CONTROL [conn205707] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:49:51.451Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:49:51.461Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:50:11.460Z I - [conn206330] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:50:11.470Z I CONTROL [conn206330] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:50:31.459Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:50:31.469Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:50:51.457Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:50:51.467Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:51:11.456Z I - [conn207413] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:51:11.465Z I CONTROL [conn207413] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:51:31.461Z I - [conn205487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:51:31.471Z I CONTROL [conn205487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:51:51.457Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:51:51.467Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:52:11.457Z I - [conn183103] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:52:11.467Z I CONTROL [conn183103] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:52:31.457Z I - [conn215139] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:52:31.467Z I CONTROL [conn215139] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:52:51.467Z I - [conn207412] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:52:51.477Z I CONTROL [conn207412] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:53:11.454Z I - [conn215971] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:53:11.464Z I CONTROL [conn215971] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:53:31.448Z I - [conn216102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:53:31.457Z I CONTROL [conn216102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:53:51.451Z I - [conn205752] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:53:51.461Z I CONTROL [conn205752] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:54:11.447Z I - [conn205928] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:54:11.457Z I CONTROL [conn205928] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:54:31.456Z I - [conn183087] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:54:31.465Z I CONTROL [conn183087] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:54:51.447Z I - [conn216052] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:54:51.457Z I CONTROL [conn216052] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:55:11.458Z I - [conn205796] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:55:11.468Z I CONTROL [conn205796] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:55:31.458Z I - [conn206771] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:55:31.468Z I CONTROL [conn206771] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:55:51.467Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:55:51.477Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:56:11.446Z I - [conn216135] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:56:11.456Z I CONTROL [conn216135] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:56:31.454Z I - [conn206771] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:56:31.464Z I CONTROL [conn206771] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:56:51.452Z I - [conn205615] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:56:51.461Z I CONTROL [conn205615] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:57:11.454Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:57:11.464Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:57:31.444Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:57:31.454Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:57:51.447Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:57:51.457Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:58:11.454Z I - [conn183114] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:58:11.464Z I CONTROL [conn183114] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:58:31.460Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:58:31.469Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:58:51.452Z I - [conn207413] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:58:51.462Z I CONTROL [conn207413] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:59:11.451Z I - [conn205876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:59:11.461Z I CONTROL [conn205876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:59:31.450Z I - [conn215982] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:59:31.459Z I CONTROL [conn215982] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T01:59:49.406Z I INDEX [conn215568] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T01:59:51.457Z I - [conn216135] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T01:59:51.466Z I CONTROL [conn216135] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:00:11.454Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:00:11.463Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:00:31.452Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:00:31.461Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:00:51.453Z I - [conn206800] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:00:51.462Z I CONTROL [conn206800] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:01:11.444Z I - [conn215960] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:01:11.453Z I CONTROL [conn215960] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:01:31.451Z I - [conn215927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:01:31.460Z I CONTROL [conn215927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:01:51.448Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:01:51.458Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:02:11.442Z I - [conn205487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:02:11.451Z I CONTROL [conn205487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:02:31.452Z I - [conn205473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:02:31.462Z I CONTROL [conn205473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:02:51.446Z I - [conn205858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:02:51.455Z I CONTROL [conn205858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:03:11.442Z I - [conn216097] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:03:11.451Z I CONTROL [conn216097] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:03:31.451Z I - [conn215143] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:03:31.460Z I CONTROL [conn215143] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:03:51.440Z I - [conn177467] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:03:51.450Z I CONTROL [conn177467] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:04:11.453Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:04:11.462Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:04:31.444Z I - [conn207414] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:04:31.454Z I CONTROL [conn207414] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:04:51.454Z I - [conn205691] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:04:51.463Z I CONTROL [conn205691] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:05:11.460Z I - [conn205691] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:05:11.470Z I CONTROL [conn205691] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:05:31.440Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:05:31.450Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:05:51.454Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:05:51.463Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:06:11.443Z I - [conn205858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:06:11.453Z I CONTROL [conn205858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:06:31.442Z I - [conn205473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:06:31.451Z I CONTROL [conn205473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:06:51.456Z I - [conn216124] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:06:51.465Z I CONTROL [conn216124] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:07:11.447Z I - [conn205707] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:07:11.457Z I CONTROL [conn205707] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:07:31.443Z I - [conn205633] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:07:31.452Z I CONTROL [conn205633] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:07:51.440Z I - [conn205933] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:07:51.450Z I CONTROL [conn205933] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:08:11.452Z I - [conn205626] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:08:11.462Z I CONTROL [conn205626] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:08:29.441Z I - [conn206298] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:08:29.451Z I CONTROL [conn206298] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:08:31.446Z I - [conn205726] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:08:31.456Z I CONTROL [conn205726] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:08:51.453Z I - [conn183087] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:08:51.463Z I CONTROL [conn183087] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:09:11.452Z I - [conn205915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:09:11.461Z I CONTROL [conn205915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:09:31.447Z I - [conn205453] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:09:31.457Z I CONTROL [conn205453] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:09:51.451Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:09:51.460Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:10:11.447Z I - [conn177379] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:10:11.457Z I CONTROL [conn177379] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:10:31.441Z I - [conn177379] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:10:31.451Z I CONTROL [conn177379] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:10:51.448Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:10:51.458Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:11:11.459Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:11:11.468Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:11:31.445Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:11:31.455Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:11:51.452Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:11:51.461Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:12:11.443Z I - [conn215957] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:12:11.453Z I CONTROL [conn215957] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:12:31.445Z I - [conn215975] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:12:31.455Z I CONTROL [conn215975] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:12:51.441Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:12:51.450Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:13:11.443Z I - [conn215929] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:13:11.452Z I CONTROL [conn215929] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:13:31.444Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:13:31.454Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:13:51.458Z I - [conn205675] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:13:51.468Z I CONTROL [conn205675] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:14:11.444Z I - [conn205863] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:14:11.453Z I CONTROL [conn205863] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:14:31.472Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:14:31.481Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:14:51.438Z I - [conn205570] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:14:51.447Z I CONTROL [conn205570] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:15:11.445Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:15:11.454Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:15:31.433Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:15:31.443Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:15:51.446Z I - [conn215755] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:15:51.455Z I CONTROL [conn215755] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:16:11.443Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:16:11.453Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:16:31.432Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:16:31.442Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:16:51.443Z I - [conn215966] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:16:51.453Z I CONTROL [conn215966] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:17:11.441Z I - [conn205857] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:17:11.451Z I CONTROL [conn205857] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:17:31.435Z I - [conn215139] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:17:31.445Z I CONTROL [conn215139] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:17:51.446Z I - [conn206800] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:17:51.456Z I CONTROL [conn206800] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:18:11.444Z I - [conn205568] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:18:11.453Z I CONTROL [conn205568] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:18:31.460Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:18:31.469Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:18:51.449Z I - [conn205949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:18:51.459Z I CONTROL [conn205949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:19:11.433Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:19:11.443Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:19:31.437Z I - [conn215855] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:19:31.447Z I CONTROL [conn215855] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:19:51.452Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:19:51.461Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:20:11.437Z I - [conn205644] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:20:11.446Z I CONTROL [conn205644] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:20:31.438Z I - [conn205644] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:20:31.448Z I CONTROL [conn205644] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:20:51.445Z I - [conn207405] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:20:51.454Z I CONTROL [conn207405] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:21:11.454Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:21:11.463Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:21:31.437Z I - [conn183103] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:21:31.446Z I CONTROL [conn183103] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:21:51.437Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:21:51.446Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:22:11.439Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:22:11.449Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:22:31.435Z I - [conn177379] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:22:31.445Z I CONTROL [conn177379] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:22:51.447Z I - [conn177379] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:22:51.457Z I CONTROL [conn177379] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:23:11.448Z I - [conn216110] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:23:11.457Z I CONTROL [conn216110] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:23:31.441Z I - [conn205499] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:23:31.451Z I CONTROL [conn205499] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:23:51.437Z I - [conn206794] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:23:51.446Z I CONTROL [conn206794] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:24:11.439Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:24:11.449Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:24:31.440Z I - [conn205586] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:24:31.450Z I CONTROL [conn205586] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:24:51.447Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:24:51.457Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:25:11.441Z I - [conn216110] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:25:11.451Z I CONTROL [conn216110] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:25:31.434Z I - [conn205915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:25:31.444Z I CONTROL [conn205915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:25:51.440Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:25:51.450Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:26:11.437Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:26:11.447Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:26:31.436Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:26:31.445Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:26:51.442Z I - [conn206771] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:26:51.451Z I CONTROL [conn206771] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:27:11.430Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:27:11.440Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:27:31.437Z I - [conn205901] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:27:31.447Z I CONTROL [conn205901] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:27:51.440Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:27:51.450Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:28:11.442Z I - [conn205937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:28:11.451Z I CONTROL [conn205937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:28:31.439Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:28:31.449Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:28:51.439Z I - [conn212225] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:28:51.449Z I CONTROL [conn212225] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:29:11.433Z I - [conn206257] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:29:11.442Z I CONTROL [conn206257] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:29:31.433Z I - [conn216117] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:29:31.442Z I CONTROL [conn216117] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:29:51.436Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:29:51.446Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:30:11.428Z I - [conn215953] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:30:11.438Z I CONTROL [conn215953] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:30:31.431Z I - [conn205585] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:30:31.440Z I CONTROL [conn205585] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:30:51.430Z I - [conn205595] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:30:51.440Z I CONTROL [conn205595] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:31:11.431Z I - [conn207413] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:31:11.441Z I CONTROL [conn207413] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:31:31.440Z I - [conn178530] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:31:31.449Z I CONTROL [conn178530] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:31:51.432Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:31:51.441Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:32:11.442Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:32:11.452Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:32:31.439Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:32:31.449Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:32:51.436Z I - [conn205607] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:32:51.446Z I CONTROL [conn205607] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:33:11.436Z I - [conn205945] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:33:11.446Z I CONTROL [conn205945] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:33:31.435Z I - [conn206800] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:33:31.445Z I CONTROL [conn206800] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:33:51.433Z I - [conn215938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:33:51.443Z I CONTROL [conn215938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:34:11.424Z I - [conn215938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:34:11.433Z I CONTROL [conn215938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:34:31.429Z I - [conn216156] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:34:31.439Z I CONTROL [conn216156] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:34:51.445Z I - [conn205598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:34:51.455Z I CONTROL [conn205598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:35:11.439Z I - [conn177467] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:35:11.449Z I CONTROL [conn177467] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:35:31.439Z I - [conn215140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:35:31.448Z I CONTROL [conn215140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:35:51.434Z I - [conn205901] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:35:51.444Z I CONTROL [conn205901] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:36:11.423Z I - [conn216095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:36:11.433Z I CONTROL [conn216095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:36:31.433Z I - [conn205816] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:36:31.443Z I CONTROL [conn205816] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:36:51.433Z I - [conn205801] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:36:51.443Z I CONTROL [conn205801] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:37:11.436Z I - [conn206318] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:37:11.445Z I CONTROL [conn206318] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:37:31.435Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:37:31.444Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:37:51.437Z I - [conn205918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:37:51.447Z I CONTROL [conn205918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:38:11.427Z I - [conn205874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:38:11.437Z I CONTROL [conn205874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:38:31.441Z I - [conn215965] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:38:31.450Z I CONTROL [conn215965] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:38:51.431Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:38:51.441Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:39:11.426Z I - [conn205791] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:39:11.435Z I CONTROL [conn205791] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:39:31.424Z I - [conn206319] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:39:31.433Z I CONTROL [conn206319] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:39:51.448Z I - [conn215977] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:39:51.458Z I CONTROL [conn215977] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:40:11.426Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:40:11.436Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:40:31.440Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:40:31.450Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:40:51.426Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:40:51.436Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:41:11.433Z I - [conn215927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:41:11.442Z I CONTROL [conn215927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:41:31.433Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:41:31.442Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:41:51.431Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:41:51.440Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:42:11.431Z I - [conn205752] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:42:11.441Z I CONTROL [conn205752] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:42:20.095Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:42:20.105Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:42:31.430Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:42:31.440Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:42:51.439Z I - [conn183336] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:42:51.448Z I CONTROL [conn183336] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:43:11.431Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:43:11.440Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:43:31.422Z I - [conn215982] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:43:31.432Z I CONTROL [conn215982] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:43:51.451Z I - [conn205876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:43:51.460Z I CONTROL [conn205876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:44:11.433Z I - [conn205671] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:44:11.443Z I CONTROL [conn205671] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:44:31.425Z I - [conn205787] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:44:31.435Z I CONTROL [conn205787] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:44:51.432Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:44:51.442Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:45:11.433Z I - [conn205945] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:45:11.443Z I CONTROL [conn205945] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:45:31.425Z I - [conn206328] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:45:31.434Z I CONTROL [conn206328] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:45:51.431Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:45:51.440Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:46:11.432Z I - [conn205857] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:46:11.441Z I CONTROL [conn205857] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:46:31.434Z I - [conn205949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:46:31.443Z I CONTROL [conn205949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:46:51.429Z I - [conn215944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:46:51.439Z I CONTROL [conn215944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:47:11.430Z I - [conn216136] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:47:11.440Z I CONTROL [conn216136] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:47:31.420Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:47:31.430Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:47:51.433Z I - [conn205804] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:47:51.443Z I CONTROL [conn205804] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:48:11.435Z I - [conn205942] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:48:11.444Z I CONTROL [conn205942] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:48:31.423Z I - [conn206771] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:48:31.433Z I CONTROL [conn206771] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:48:51.419Z I - [conn206794] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:48:51.428Z I CONTROL [conn206794] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:49:11.430Z I - [conn206330] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:49:11.439Z I CONTROL [conn206330] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:49:31.428Z I - [conn170812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:49:31.438Z I CONTROL [conn170812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:49:51.425Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:49:51.435Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:50:11.428Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:50:11.438Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:50:31.417Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:50:31.426Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:50:51.421Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:50:51.430Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:51:11.426Z I - [conn205949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:51:11.436Z I CONTROL [conn205949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:51:31.420Z I - [conn205801] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:51:31.430Z I CONTROL [conn205801] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:51:51.426Z I - [conn216062] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:51:51.435Z I CONTROL [conn216062] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:52:11.430Z I - [conn205801] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:52:11.440Z I CONTROL [conn205801] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:52:31.437Z I - [conn215957] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:52:31.447Z I CONTROL [conn215957] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:52:51.427Z I - [conn212225] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:52:51.437Z I CONTROL [conn212225] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:53:11.432Z I - [conn215134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:53:11.442Z I CONTROL [conn215134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:53:31.429Z I - [conn216125] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:53:31.438Z I CONTROL [conn216125] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:53:51.426Z I - [conn205572] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:53:51.436Z I CONTROL [conn205572] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:54:11.421Z I - [conn205457] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:54:11.431Z I CONTROL [conn205457] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:54:20.745Z I - [conn216125] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:54:20.755Z I CONTROL [conn216125] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:54:31.430Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:54:31.439Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:54:51.431Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:54:51.441Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:55:11.417Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:55:11.426Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:55:31.424Z I - [conn215911] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:55:31.434Z I CONTROL [conn215911] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:55:51.428Z I - [conn216147] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:55:51.437Z I CONTROL [conn216147] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:56:11.426Z I - [conn205814] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:56:11.435Z I CONTROL [conn205814] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:56:31.425Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:56:31.434Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:56:51.431Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:56:51.441Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:57:11.416Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:57:11.426Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:57:31.433Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:57:31.443Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:57:51.428Z I - [conn215855] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:57:51.437Z I CONTROL [conn215855] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:58:11.439Z I - [conn215145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:58:11.449Z I CONTROL [conn215145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:58:31.424Z I - [conn205945] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:58:31.434Z I CONTROL [conn205945] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:58:51.415Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:58:51.425Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:59:11.426Z I - [conn206813] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:59:11.436Z I CONTROL [conn206813] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:59:31.426Z I - [conn212596] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:59:31.436Z I CONTROL [conn212596] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T02:59:49.288Z I INDEX [conn215461] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T02:59:51.426Z I - [conn205942] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T02:59:51.436Z I CONTROL [conn205942] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:00:11.423Z I - [conn215971] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:00:11.433Z I CONTROL [conn215971] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:00:31.433Z I - [conn206811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:00:31.443Z I CONTROL [conn206811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:00:51.429Z I - [conn205617] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:00:51.439Z I CONTROL [conn205617] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:01:11.433Z I - [conn212229] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:01:11.442Z I CONTROL [conn212229] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:01:31.423Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:01:31.433Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:01:51.425Z I - [conn170487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:01:51.435Z I CONTROL [conn170487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:02:11.428Z I - [conn205858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:02:11.437Z I CONTROL [conn205858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:02:31.422Z I - [conn206319] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:02:31.432Z I CONTROL [conn206319] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:02:51.425Z I - [conn206319] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:02:51.434Z I CONTROL [conn206319] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:03:11.424Z I - [conn205585] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:03:11.433Z I CONTROL [conn205585] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:03:31.425Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:03:31.434Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:03:51.424Z I - [conn205942] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:03:51.434Z I CONTROL [conn205942] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:04:11.411Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:04:11.421Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:04:31.422Z I - [conn205948] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:04:31.432Z I CONTROL [conn205948] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:04:51.413Z I - [conn205808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:04:51.422Z I CONTROL [conn205808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:05:11.423Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:05:11.433Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:05:31.424Z I - [conn216124] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:05:31.434Z I CONTROL [conn216124] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:05:51.424Z I - [conn205471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:05:51.433Z I CONTROL [conn205471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:06:11.423Z I - [conn177379] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:06:11.432Z I CONTROL [conn177379] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:06:31.425Z I - [conn205815] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:06:31.434Z I CONTROL [conn205815] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:06:51.420Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:06:51.429Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:07:11.437Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:07:11.446Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:07:31.410Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:07:31.419Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:07:51.410Z I - [conn206797] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:07:51.420Z I CONTROL [conn206797] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:08:11.421Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:08:11.430Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:08:31.434Z I - [conn205876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:08:31.443Z I CONTROL [conn205876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:08:51.423Z I - [conn205499] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:08:51.433Z I CONTROL [conn205499] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:09:11.430Z I - [conn215930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:09:11.439Z I CONTROL [conn215930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:09:31.409Z I - [conn205930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:09:31.418Z I CONTROL [conn205930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:09:51.412Z I - [conn205930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:09:51.422Z I CONTROL [conn205930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:10:11.420Z I - [conn215962] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:10:11.430Z I CONTROL [conn215962] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:10:31.412Z I - [conn177379] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:10:31.421Z I CONTROL [conn177379] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:10:51.420Z I - [conn215140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:10:51.430Z I CONTROL [conn215140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:11:11.414Z I - [conn183171] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:11:11.423Z I CONTROL [conn183171] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:11:31.416Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:11:31.426Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:11:51.409Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:11:51.418Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:12:11.411Z I - [conn170812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:12:11.421Z I CONTROL [conn170812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:12:31.406Z I - [conn215966] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:12:31.416Z I CONTROL [conn215966] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:12:51.420Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:12:51.429Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:13:11.424Z I - [conn170487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:13:11.433Z I CONTROL [conn170487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:13:31.419Z I - [conn206311] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:13:31.429Z I CONTROL [conn206311] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:13:51.410Z I - [conn170487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:13:51.420Z I CONTROL [conn170487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:14:11.409Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:14:11.418Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:14:31.417Z I - [conn215876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:14:31.427Z I CONTROL [conn215876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:14:51.421Z I - [conn216118] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:14:51.431Z I CONTROL [conn216118] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:15:11.420Z I - [conn207414] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:15:11.430Z I CONTROL [conn207414] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:15:31.426Z I - [conn216125] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:15:31.436Z I CONTROL [conn216125] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:15:51.408Z I - [conn206330] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:15:51.418Z I CONTROL [conn206330] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:16:11.414Z I - [conn170487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:16:11.423Z I CONTROL [conn170487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:16:31.427Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:16:31.437Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:16:51.406Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:16:51.416Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:17:11.416Z I - [conn215947] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:17:11.426Z I CONTROL [conn215947] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:17:31.407Z I - [conn215947] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:17:31.416Z I CONTROL [conn215947] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:17:51.427Z I - [conn205518] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:17:51.437Z I CONTROL [conn205518] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:18:11.420Z I - [conn215143] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:18:11.430Z I CONTROL [conn215143] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:18:31.415Z I - [conn215960] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:18:31.425Z I CONTROL [conn215960] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:18:51.416Z I - [conn205586] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:18:51.426Z I CONTROL [conn205586] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:19:11.416Z I - [conn205822] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:19:11.425Z I CONTROL [conn205822] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:19:31.403Z I - [conn216131] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:19:31.413Z I CONTROL [conn216131] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:19:51.415Z I - [conn216128] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:19:51.424Z I CONTROL [conn216128] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:20:11.416Z I - [conn216097] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:20:11.426Z I CONTROL [conn216097] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:20:31.416Z I - [conn216097] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:20:31.426Z I CONTROL [conn216097] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:20:51.416Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:20:51.425Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:21:11.409Z I - [conn215947] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:21:11.419Z I CONTROL [conn215947] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:21:31.423Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:21:31.432Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:21:51.416Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:21:51.426Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:22:11.411Z I - [conn183343] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:22:11.421Z I CONTROL [conn183343] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:22:31.413Z I - [conn183343] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:22:31.422Z I CONTROL [conn183343] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:22:51.416Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:22:51.426Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:23:11.406Z I - [conn207402] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:23:11.416Z I CONTROL [conn207402] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:23:31.414Z I - [conn206800] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:23:31.424Z I CONTROL [conn206800] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:23:51.415Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:23:51.424Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:24:11.410Z I - [conn215943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:24:11.419Z I CONTROL [conn215943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:24:31.413Z I - [conn215960] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:24:31.423Z I CONTROL [conn215960] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:24:51.411Z I - [conn183429] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:24:51.421Z I CONTROL [conn183429] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:25:11.411Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:25:11.421Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:25:31.404Z I - [conn205876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:25:31.414Z I CONTROL [conn205876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:25:51.417Z I - [conn206316] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:25:51.426Z I CONTROL [conn206316] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:26:11.402Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:26:11.412Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:26:31.401Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:26:31.410Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:26:51.404Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:26:51.413Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:27:11.412Z I - [conn205457] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:27:11.422Z I CONTROL [conn205457] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:27:31.407Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:27:31.417Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:27:51.410Z I - [conn170487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:27:51.420Z I CONTROL [conn170487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:28:11.415Z I - [conn206318] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:28:11.425Z I CONTROL [conn206318] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:28:31.416Z I - [conn205749] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:28:31.426Z I CONTROL [conn205749] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:28:51.433Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:28:51.443Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:29:11.401Z I - [conn216146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:29:11.411Z I CONTROL [conn216146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:29:31.412Z I - [conn183114] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:29:31.422Z I CONTROL [conn183114] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:29:51.424Z I - [conn183104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:29:51.434Z I CONTROL [conn183104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:30:11.413Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:30:11.423Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:30:31.412Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:30:31.422Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:30:51.412Z I - [conn170472] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:30:51.421Z I CONTROL [conn170472] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:31:11.593Z I - [conn212596] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:31:11.603Z I CONTROL [conn212596] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:31:31.413Z I - [conn216123] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:31:31.422Z I CONTROL [conn216123] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:31:51.408Z I - [conn205933] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:31:51.417Z I CONTROL [conn205933] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:32:11.416Z I - [conn206328] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:32:11.426Z I CONTROL [conn206328] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:32:31.403Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:32:31.412Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:32:51.414Z I - [conn206329] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:32:51.423Z I CONTROL [conn206329] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:33:11.422Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:33:11.432Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:33:31.429Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:33:31.439Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:33:51.409Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:33:51.418Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:34:11.413Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:34:11.423Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:34:31.409Z I - [conn216112] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:34:31.419Z I CONTROL [conn216112] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:34:51.412Z I - [conn215909] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:34:51.422Z I CONTROL [conn215909] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:35:11.409Z I - [conn170471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:35:11.419Z I CONTROL [conn170471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:35:31.403Z I - [conn216129] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:35:31.413Z I CONTROL [conn216129] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:35:51.412Z I - [conn205585] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:35:51.422Z I CONTROL [conn205585] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:36:11.412Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:36:11.422Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:36:31.407Z I - [conn205644] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:36:31.416Z I CONTROL [conn205644] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:36:51.413Z I - [conn205499] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:36:51.422Z I CONTROL [conn205499] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:37:11.410Z I - [conn205587] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:37:11.419Z I CONTROL [conn205587] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:37:31.409Z I - [conn206319] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:37:31.419Z I CONTROL [conn206319] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:37:51.401Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:37:51.410Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:38:11.411Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:38:11.420Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:38:31.406Z I - [conn205863] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:38:31.416Z I CONTROL [conn205863] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:38:51.407Z I - [conn207408] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:38:51.417Z I CONTROL [conn207408] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:39:11.406Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:39:11.416Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:39:31.406Z I - [conn205671] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:39:31.416Z I CONTROL [conn205671] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:39:51.409Z I - [conn206329] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:39:51.419Z I CONTROL [conn206329] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:40:11.409Z I - [conn205749] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:40:11.419Z I CONTROL [conn205749] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:40:31.397Z I - [conn205844] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:40:31.407Z I CONTROL [conn205844] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:40:51.408Z I - [conn205572] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:40:51.418Z I CONTROL [conn205572] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:41:11.395Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:41:11.405Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:41:31.406Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:41:31.416Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:41:51.413Z I - [conn205791] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:41:51.423Z I CONTROL [conn205791] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:42:11.407Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:42:11.416Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:42:31.406Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:42:31.416Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:42:51.424Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:42:51.434Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:43:11.406Z I - [conn205787] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:43:11.415Z I CONTROL [conn205787] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:43:31.403Z I - [conn216136] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:43:31.413Z I CONTROL [conn216136] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:43:51.406Z I - [conn216153] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:43:51.416Z I CONTROL [conn216153] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:44:11.393Z I - [conn205457] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:44:11.403Z I CONTROL [conn205457] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:44:31.405Z I - [conn216097] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:44:31.415Z I CONTROL [conn216097] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:44:51.415Z I - [conn205424] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:44:51.425Z I CONTROL [conn205424] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:45:11.402Z I - [conn216107] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:45:11.411Z I CONTROL [conn216107] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:45:31.405Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:45:31.414Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:45:51.403Z I - [conn206797] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:45:51.413Z I CONTROL [conn206797] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:46:11.401Z I - [conn205656] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:46:11.410Z I CONTROL [conn205656] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:46:31.392Z I - [conn205656] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:46:31.402Z I CONTROL [conn205656] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:46:51.407Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:46:51.417Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:47:11.404Z I - [conn205937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:47:11.414Z I CONTROL [conn205937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:47:31.407Z I - [conn205707] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:47:31.416Z I CONTROL [conn205707] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:47:51.407Z I - [conn183104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:47:51.416Z I CONTROL [conn183104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:48:11.398Z I - [conn215943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:48:11.408Z I CONTROL [conn215943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:48:31.402Z I - [conn216125] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:48:31.411Z I CONTROL [conn216125] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:48:51.396Z I - [conn215966] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:48:51.406Z I CONTROL [conn215966] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:49:11.401Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:49:11.410Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:49:31.400Z I - [conn207402] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:49:31.410Z I CONTROL [conn207402] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:49:51.402Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:49:51.411Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:50:11.392Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:50:11.402Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:50:31.414Z I - [conn183114] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:50:31.424Z I CONTROL [conn183114] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:50:51.402Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:50:51.412Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:51:11.404Z I - [conn177467] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:51:11.414Z I CONTROL [conn177467] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:51:31.392Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:51:31.402Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:51:51.398Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:51:51.407Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:52:11.402Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:52:11.412Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:52:31.398Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:52:31.410Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:52:51.386Z I - [conn215919] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:52:51.396Z I CONTROL [conn215919] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:53:11.416Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:53:11.426Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:53:31.389Z I - [conn206321] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:53:31.399Z I CONTROL [conn206321] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:53:51.394Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:53:51.404Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:54:11.403Z I - [conn205517] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:54:11.413Z I CONTROL [conn205517] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:54:31.400Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:54:31.410Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:54:51.395Z I - [conn216136] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:54:51.405Z I CONTROL [conn216136] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:55:11.393Z I - [conn215976] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:55:11.404Z I CONTROL [conn215976] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:55:31.388Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:55:31.398Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:55:51.400Z I - [conn205930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:55:51.410Z I CONTROL [conn205930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:56:11.394Z I - [conn215976] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:56:11.403Z I CONTROL [conn215976] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:56:31.390Z I - [conn207405] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:56:31.400Z I CONTROL [conn207405] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:56:51.396Z I - [conn205607] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:56:51.406Z I CONTROL [conn205607] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:57:11.398Z I - [conn216135] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:57:11.408Z I CONTROL [conn216135] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:57:31.397Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:57:31.407Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:57:51.396Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:57:51.406Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:58:11.394Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:58:11.403Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:58:31.387Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:58:31.397Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:58:51.390Z I - [conn205872] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:58:51.400Z I CONTROL [conn205872] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:59:11.388Z I - [conn205787] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:59:11.397Z I CONTROL [conn205787] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:59:31.396Z I - [conn205798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:59:31.406Z I CONTROL [conn205798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T03:59:51.286Z I INDEX [conn215568] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T03:59:51.393Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T03:59:51.403Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:00:11.393Z I - [conn216131] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:00:11.403Z I CONTROL [conn216131] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:00:31.394Z I - [conn215145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:00:31.403Z I CONTROL [conn215145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:00:51.387Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:00:51.397Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:01:11.388Z I - [conn205844] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:01:11.398Z I CONTROL [conn205844] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:01:31.394Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:01:31.404Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:01:51.438Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:01:51.448Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:02:11.385Z I - [conn207408] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:02:11.395Z I CONTROL [conn207408] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:02:31.407Z I - [conn205874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:02:31.416Z I CONTROL [conn205874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:02:51.393Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:02:51.402Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:03:11.393Z I - [conn170471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:03:11.402Z I CONTROL [conn170471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:03:31.392Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:03:31.402Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:03:51.393Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:03:51.403Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:04:11.398Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:04:11.408Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:04:31.382Z I - [conn216069] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:04:31.392Z I CONTROL [conn216069] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:04:51.393Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:04:51.402Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:05:11.395Z I - [conn205607] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:05:11.404Z I CONTROL [conn205607] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:05:31.391Z I - [conn183115] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:05:31.400Z I CONTROL [conn183115] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:05:51.397Z I - [conn207413] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:05:51.407Z I CONTROL [conn207413] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:06:11.393Z I - [conn207414] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:06:11.403Z I CONTROL [conn207414] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:06:31.408Z I - [conn205853] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:06:31.418Z I CONTROL [conn205853] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:06:51.382Z I - [conn205815] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:06:51.391Z I CONTROL [conn205815] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:07:11.393Z I - [conn212229] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:07:11.403Z I CONTROL [conn212229] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:07:31.392Z I - [conn170812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:07:31.402Z I CONTROL [conn170812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:07:51.396Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:07:51.405Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:08:11.386Z I - [conn205656] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:08:11.395Z I CONTROL [conn205656] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:08:31.386Z I - [conn215143] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:08:31.396Z I CONTROL [conn215143] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:08:40.390Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:08:40.400Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:08:51.391Z I - [conn205656] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:08:51.401Z I CONTROL [conn205656] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:09:11.379Z I - [conn215976] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:09:11.388Z I CONTROL [conn215976] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:09:31.395Z I - [conn215911] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:09:31.405Z I CONTROL [conn215911] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:09:51.392Z I - [conn205844] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:09:51.401Z I CONTROL [conn205844] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:10:11.395Z I - [conn215140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:10:11.404Z I CONTROL [conn215140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:10:31.394Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:10:31.404Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:10:51.388Z I - [conn215938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:10:51.397Z I CONTROL [conn215938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:11:11.395Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:11:11.404Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:11:31.384Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:11:31.394Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:11:51.380Z I - [conn215975] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:11:51.390Z I CONTROL [conn215975] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:12:11.398Z I - [conn216110] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:12:11.407Z I CONTROL [conn216110] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:12:31.392Z I - [conn205872] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:12:31.401Z I CONTROL [conn205872] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:12:51.386Z I - [conn205671] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:12:51.395Z I CONTROL [conn205671] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:13:11.390Z I - [conn215850] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:13:11.400Z I CONTROL [conn215850] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:13:31.399Z I - [conn212229] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:13:31.409Z I CONTROL [conn212229] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:13:51.391Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:13:51.401Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:14:11.389Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:14:11.399Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:14:31.391Z I - [conn205656] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:14:31.401Z I CONTROL [conn205656] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:14:51.389Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:14:51.399Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:15:11.381Z I - [conn178530] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:15:11.391Z I CONTROL [conn178530] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:15:31.384Z I - [conn215140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:15:31.394Z I CONTROL [conn215140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:15:51.390Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:15:51.400Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:16:11.400Z I - [conn206771] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:16:11.410Z I CONTROL [conn206771] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:16:31.391Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:16:31.400Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:16:51.402Z I - [conn170487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:16:51.412Z I CONTROL [conn170487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:17:11.383Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:17:11.393Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:17:31.388Z I - [conn215930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:17:31.397Z I CONTROL [conn215930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:17:51.393Z I - [conn205633] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:17:51.403Z I CONTROL [conn205633] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:18:11.408Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:18:11.417Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:18:31.391Z I - [conn206257] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:18:31.401Z I CONTROL [conn206257] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:18:51.385Z I - [conn215876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:18:51.394Z I CONTROL [conn215876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:19:11.381Z I - [conn183114] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:19:11.391Z I CONTROL [conn183114] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:19:31.629Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:19:31.638Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:19:51.388Z I - [conn215755] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:19:51.398Z I CONTROL [conn215755] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:20:11.387Z I - [conn216102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:20:11.396Z I CONTROL [conn216102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:20:31.387Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:20:31.396Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:20:51.389Z I - [conn207412] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:20:51.399Z I CONTROL [conn207412] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:21:11.377Z I - [conn205822] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:21:11.387Z I CONTROL [conn205822] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:21:31.402Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:21:31.412Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:21:51.386Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:21:51.395Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:22:11.392Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:22:11.402Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:22:31.395Z I - [conn215963] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:22:31.405Z I CONTROL [conn215963] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:22:51.391Z I - [conn205878] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:22:51.400Z I CONTROL [conn205878] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:23:11.386Z I - [conn205878] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:23:11.395Z I CONTROL [conn205878] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:23:31.386Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:23:31.396Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:23:51.396Z I - [conn205598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:23:51.405Z I CONTROL [conn205598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:24:11.385Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:24:11.395Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:24:31.388Z I - [conn205598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:24:31.397Z I CONTROL [conn205598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:24:51.227Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:24:51.237Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:24:51.385Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:24:51.395Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:25:11.391Z I - [conn205935] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:25:11.401Z I CONTROL [conn205935] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:25:31.389Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:25:31.398Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:25:51.397Z I - [conn205857] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:25:51.408Z I CONTROL [conn205857] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:26:11.375Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:26:11.384Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:26:31.385Z I - [conn216147] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:26:31.395Z I CONTROL [conn216147] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:26:51.387Z I - [conn215965] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:26:51.396Z I CONTROL [conn215965] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:27:11.386Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:27:11.395Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:27:31.376Z I - [conn205816] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:27:31.385Z I CONTROL [conn205816] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:27:51.385Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:27:51.395Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:28:11.384Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:28:11.393Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:28:31.385Z I - [conn206298] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:28:31.395Z I CONTROL [conn206298] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:28:51.385Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:28:51.395Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:29:11.386Z I - [conn183429] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:29:11.397Z I CONTROL [conn183429] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:29:31.386Z I - [conn206807] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:29:31.395Z I CONTROL [conn206807] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:29:51.380Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:29:51.389Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:30:11.395Z I - [conn216129] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:30:11.405Z I CONTROL [conn216129] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:30:31.386Z I - [conn183171] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:30:31.396Z I CONTROL [conn183171] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:30:51.385Z I - [conn206771] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:30:51.395Z I CONTROL [conn206771] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:31:11.378Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:31:11.387Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:31:31.375Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:31:31.384Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:31:51.385Z I - [conn205586] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:31:51.394Z I CONTROL [conn205586] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:32:11.384Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:32:11.393Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:32:31.378Z I - [conn205814] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:32:31.388Z I CONTROL [conn205814] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:32:51.382Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:32:51.391Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:33:11.382Z I - [conn205863] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:33:11.391Z I CONTROL [conn205863] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:33:31.383Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:33:31.393Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:33:51.379Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:33:51.389Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:34:11.390Z I - [conn215982] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:34:11.400Z I CONTROL [conn215982] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:34:31.376Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:34:31.385Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:34:51.395Z I - [conn205644] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:34:51.404Z I CONTROL [conn205644] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:35:11.379Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:35:11.388Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:35:31.378Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:35:31.388Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:35:51.385Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:35:51.394Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:36:11.382Z I - [conn216147] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:36:11.392Z I CONTROL [conn216147] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:36:31.378Z I - [conn215850] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:36:31.388Z I CONTROL [conn215850] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:36:51.381Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:36:51.391Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:37:11.381Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:37:11.390Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:37:31.378Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:37:31.388Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:37:51.384Z I - [conn207412] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:37:51.393Z I CONTROL [conn207412] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:38:11.382Z I - [conn206771] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:38:11.392Z I CONTROL [conn206771] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:38:31.381Z I - [conn216144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:38:31.390Z I CONTROL [conn216144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:38:51.392Z I - [conn215968] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:38:51.402Z I CONTROL [conn215968] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:39:11.381Z I - [conn206807] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:39:11.391Z I CONTROL [conn206807] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:39:31.380Z I - [conn215966] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:39:31.390Z I CONTROL [conn215966] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:39:51.381Z I - [conn216156] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:39:51.391Z I CONTROL [conn216156] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:40:11.381Z I - [conn215145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:40:11.390Z I CONTROL [conn215145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:40:31.384Z I - [conn206321] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:40:31.394Z I CONTROL [conn206321] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:40:51.374Z I - [conn205639] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:40:51.383Z I CONTROL [conn205639] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:41:11.383Z I - [conn205743] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:41:11.393Z I CONTROL [conn205743] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:41:31.377Z I - [conn216102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:41:31.387Z I CONTROL [conn216102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:41:51.378Z I - [conn215858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:41:51.388Z I CONTROL [conn215858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:42:11.382Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:42:11.392Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:42:31.377Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:42:31.387Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:42:51.378Z I - [conn205671] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:42:51.388Z I CONTROL [conn205671] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:43:11.378Z I - [conn206319] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:43:11.387Z I CONTROL [conn206319] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:43:31.376Z I - [conn177379] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:43:31.385Z I CONTROL [conn177379] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:43:51.381Z I - [conn215943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:43:51.390Z I CONTROL [conn215943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:44:11.367Z I - [conn216095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:44:11.377Z I CONTROL [conn216095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:44:31.388Z I - [conn205517] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:44:31.398Z I CONTROL [conn205517] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:44:51.378Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:44:51.387Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:45:11.376Z I - [conn205948] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:45:11.386Z I CONTROL [conn205948] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:45:31.377Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:45:31.386Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:45:51.374Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:45:51.383Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:46:11.380Z I - [conn205638] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:46:11.389Z I CONTROL [conn205638] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:46:31.381Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:46:31.391Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:46:51.382Z I - [conn183171] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:46:51.391Z I CONTROL [conn183171] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:47:11.381Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:47:11.390Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:47:31.386Z I - [conn205749] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:47:31.396Z I CONTROL [conn205749] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:47:51.379Z I - [conn212225] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:47:51.388Z I CONTROL [conn212225] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:48:11.380Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:48:11.389Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:48:31.375Z I - [conn215139] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:48:31.385Z I CONTROL [conn215139] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:48:51.378Z I - [conn183343] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:48:51.388Z I CONTROL [conn183343] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:49:11.381Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:49:11.390Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:49:31.374Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:49:31.383Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:49:51.373Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:49:51.382Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:50:11.369Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:50:11.378Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:50:31.374Z I - [conn215929] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:50:31.384Z I CONTROL [conn215929] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:50:51.367Z I - [conn170487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:50:51.376Z I CONTROL [conn170487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:51:11.372Z I - [conn183171] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:51:11.382Z I CONTROL [conn183171] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:51:31.373Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:51:31.382Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:51:51.375Z I - [conn215927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:51:51.384Z I CONTROL [conn215927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:52:11.378Z I - [conn215927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:52:11.388Z I CONTROL [conn215927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:52:31.374Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:52:31.383Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:52:51.375Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:52:51.384Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:53:11.370Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:53:11.380Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:53:31.382Z I - [conn207414] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:53:31.392Z I CONTROL [conn207414] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:53:51.382Z I - [conn215965] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:53:51.392Z I CONTROL [conn215965] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:54:11.375Z I - [conn206321] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:54:11.384Z I CONTROL [conn206321] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:54:31.380Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:54:31.390Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:54:51.374Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:54:51.384Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:55:11.377Z I - [conn205585] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:55:11.386Z I CONTROL [conn205585] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:55:31.364Z I - [conn207414] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:55:31.374Z I CONTROL [conn207414] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:55:51.374Z I - [conn205585] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:55:51.384Z I CONTROL [conn205585] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:56:11.383Z I - [conn170812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:56:11.392Z I CONTROL [conn170812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:56:31.365Z I - [conn205585] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:56:31.375Z I CONTROL [conn205585] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:56:51.372Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:56:51.382Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:57:11.373Z I - [conn205815] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:57:11.383Z I CONTROL [conn205815] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:57:31.372Z I - [conn183114] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:57:31.382Z I CONTROL [conn183114] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:57:51.366Z I - [conn205815] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:57:51.376Z I CONTROL [conn205815] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:58:11.372Z I - [conn215982] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:58:11.381Z I CONTROL [conn215982] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:58:31.367Z I - [conn206320] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:58:31.377Z I CONTROL [conn206320] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:58:51.362Z I - [conn215943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:58:51.371Z I CONTROL [conn215943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:59:11.372Z I - [conn205876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:59:11.382Z I CONTROL [conn205876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:59:31.374Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:59:31.384Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T04:59:49.516Z I INDEX [conn215461] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T04:59:51.375Z I - [conn205587] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T04:59:51.384Z I CONTROL [conn205587] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:00:11.374Z I - [conn170471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:00:11.384Z I CONTROL [conn170471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:00:31.388Z I - [conn205920] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:00:31.398Z I CONTROL [conn205920] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:00:51.364Z I - [conn205473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:00:51.374Z I CONTROL [conn205473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:01:11.366Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:01:11.376Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:01:31.384Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:01:31.393Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:01:51.374Z I - [conn215919] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:01:51.383Z I CONTROL [conn215919] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:02:11.370Z I - [conn215965] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:02:11.380Z I CONTROL [conn215965] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:02:31.372Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:02:31.381Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:02:51.369Z I - [conn206257] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:02:51.379Z I CONTROL [conn206257] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:03:11.372Z I - [conn205796] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:03:11.382Z I CONTROL [conn205796] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:03:31.368Z I - [conn215975] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:03:31.378Z I CONTROL [conn215975] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:03:51.365Z I - [conn205844] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:03:51.374Z I CONTROL [conn205844] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:04:11.372Z I - [conn205456] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:04:11.381Z I CONTROL [conn205456] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:04:31.376Z I - [conn205844] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:04:31.385Z I CONTROL [conn205844] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:04:51.373Z I - [conn205487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:04:51.382Z I CONTROL [conn205487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:05:11.374Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:05:11.383Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:05:31.369Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:05:31.379Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:05:51.380Z I - [conn205656] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:05:51.389Z I CONTROL [conn205656] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:06:11.381Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:06:11.390Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:06:31.377Z I - [conn206319] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:06:31.386Z I CONTROL [conn206319] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:06:51.364Z I - [conn205633] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:06:51.373Z I CONTROL [conn205633] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:07:11.373Z I - [conn212596] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:07:11.382Z I CONTROL [conn212596] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:07:31.360Z I - [conn183088] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:07:31.369Z I CONTROL [conn183088] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:07:51.367Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:07:51.377Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:08:11.370Z I - [conn215943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:08:11.379Z I CONTROL [conn215943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:08:31.383Z I - [conn205424] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:08:31.393Z I CONTROL [conn205424] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:08:51.363Z I - [conn205572] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:08:51.372Z I CONTROL [conn205572] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:09:11.367Z I - [conn170472] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:09:11.376Z I CONTROL [conn170472] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:09:31.359Z I - [conn170472] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:09:31.368Z I CONTROL [conn170472] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:09:51.370Z I - [conn215913] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:09:51.380Z I CONTROL [conn215913] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:10:11.374Z I - [conn205944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:10:11.383Z I CONTROL [conn205944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:10:31.368Z I - [conn212596] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:10:31.377Z I CONTROL [conn212596] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:10:51.376Z I - [conn183171] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:10:51.385Z I CONTROL [conn183171] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:11:11.365Z I - [conn206811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:11:11.374Z I CONTROL [conn206811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:11:31.377Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:11:31.386Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:11:51.369Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:11:51.379Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:12:11.368Z I - [conn205917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:12:11.378Z I CONTROL [conn205917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:12:31.369Z I - [conn206330] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:12:31.379Z I CONTROL [conn206330] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:12:51.370Z I - [conn215931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:12:51.379Z I CONTROL [conn215931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:13:11.365Z I - [conn205822] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:13:11.375Z I CONTROL [conn205822] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:13:31.369Z I - [conn206797] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:13:31.378Z I CONTROL [conn206797] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:13:51.361Z I - [conn212229] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:13:51.371Z I CONTROL [conn212229] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:14:11.369Z I - [conn205917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:14:11.378Z I CONTROL [conn205917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:14:31.370Z I - [conn216069] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:14:31.379Z I CONTROL [conn216069] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:14:51.363Z I - [conn205796] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:14:51.373Z I CONTROL [conn205796] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:15:11.367Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:15:11.377Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:15:31.364Z I - [conn205935] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:15:31.373Z I CONTROL [conn205935] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:15:51.390Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:15:51.400Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:16:11.365Z I - [conn205853] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:16:11.374Z I CONTROL [conn205853] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:16:31.367Z I - [conn215911] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:16:31.377Z I CONTROL [conn215911] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:16:51.367Z I - [conn205822] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:16:51.376Z I CONTROL [conn205822] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:17:11.364Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:17:11.373Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:17:31.367Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:17:31.376Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:17:51.367Z I - [conn207405] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:17:51.377Z I CONTROL [conn207405] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:18:11.362Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:18:11.372Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:18:31.367Z I - [conn205644] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:18:31.376Z I CONTROL [conn205644] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:18:51.367Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:18:51.377Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:19:11.374Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:19:11.383Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:19:31.367Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:19:31.376Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:19:51.366Z I - [conn205857] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:19:51.376Z I CONTROL [conn205857] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:20:11.355Z I - [conn212225] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:20:11.364Z I CONTROL [conn212225] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:20:31.367Z I - [conn205930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:20:31.377Z I CONTROL [conn205930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:20:51.355Z I - [conn183104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:20:51.364Z I CONTROL [conn183104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:21:11.364Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:21:11.374Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:21:31.367Z I - [conn206774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:21:31.377Z I CONTROL [conn206774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:21:51.374Z I - [conn215926] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:21:51.384Z I CONTROL [conn215926] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:22:11.367Z I - [conn206774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:22:11.377Z I CONTROL [conn206774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:22:31.366Z I - [conn206774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:22:31.376Z I CONTROL [conn206774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:22:51.367Z I - [conn206311] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:22:51.376Z I CONTROL [conn206311] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:23:11.371Z I - [conn216118] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:23:11.381Z I CONTROL [conn216118] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:23:31.364Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:23:31.374Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:23:51.373Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:23:51.382Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:24:06.545Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:24:06.554Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:24:11.353Z I - [conn215966] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:24:11.363Z I CONTROL [conn215966] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:24:31.364Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:24:31.374Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:24:51.362Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:24:51.373Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:25:11.362Z I - [conn216138] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:25:11.372Z I CONTROL [conn216138] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:25:31.375Z I - [conn205928] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:25:31.385Z I CONTROL [conn205928] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:25:51.355Z I - [conn205752] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:25:51.365Z I CONTROL [conn205752] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:26:11.365Z I - [conn216119] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:26:11.375Z I CONTROL [conn216119] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:26:31.360Z I - [conn206807] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:26:31.370Z I CONTROL [conn206807] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:26:51.370Z I - [conn207413] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:26:51.380Z I CONTROL [conn207413] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:27:11.355Z I - [conn206807] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:27:11.365Z I CONTROL [conn206807] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:27:31.357Z I - [conn205517] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:27:31.367Z I CONTROL [conn205517] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:27:51.362Z I - [conn206807] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:27:51.372Z I CONTROL [conn206807] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:28:11.351Z I - [conn215139] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:28:11.361Z I CONTROL [conn215139] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:28:31.362Z I - [conn215965] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:28:31.371Z I CONTROL [conn215965] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:28:51.352Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:28:51.361Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:29:11.363Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:29:11.373Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:29:31.362Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:29:31.371Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:29:51.359Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:29:51.369Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:30:11.358Z I - [conn205617] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:30:11.367Z I CONTROL [conn205617] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:30:31.365Z I - [conn205942] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:30:31.375Z I CONTROL [conn205942] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:30:51.362Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:30:51.371Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:31:11.354Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:31:11.363Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:31:31.354Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:31:31.364Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:31:51.357Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:31:51.367Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:32:11.361Z I - [conn216110] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:32:11.370Z I CONTROL [conn216110] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:32:31.349Z I - [conn215983] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:32:31.358Z I CONTROL [conn215983] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:32:51.362Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:32:51.372Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:33:11.355Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:33:11.364Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:33:31.352Z I - [conn216132] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:33:31.362Z I CONTROL [conn216132] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:33:51.361Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:33:51.371Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:34:11.349Z I - [conn170473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:34:11.359Z I CONTROL [conn170473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:34:31.378Z I - [conn205518] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:34:31.387Z I CONTROL [conn205518] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:34:51.358Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:34:51.368Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:35:11.351Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:35:11.361Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:35:31.367Z I - [conn205656] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:35:31.377Z I CONTROL [conn205656] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:35:51.362Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:35:51.371Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:36:11.355Z I - [conn215931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:36:11.364Z I CONTROL [conn215931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:36:31.360Z I - [conn215931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:36:31.369Z I CONTROL [conn215931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:36:51.359Z I - [conn206257] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:36:51.369Z I CONTROL [conn206257] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:37:11.350Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:37:11.360Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:37:31.356Z I - [conn215957] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:37:31.366Z I CONTROL [conn215957] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:37:51.365Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:37:51.374Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:38:11.349Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:38:11.359Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:38:31.358Z I - [conn215957] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:38:31.367Z I CONTROL [conn215957] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:38:51.352Z I - [conn216102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:38:51.361Z I CONTROL [conn216102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:39:11.363Z I - [conn205517] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:39:11.373Z I CONTROL [conn205517] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:39:31.346Z I - [conn215966] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:39:31.356Z I CONTROL [conn215966] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:39:51.355Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:39:51.365Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:40:11.361Z I - [conn216147] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:40:11.371Z I CONTROL [conn216147] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:40:31.355Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:40:31.365Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:40:51.348Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:40:51.358Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:41:11.363Z I - [conn215908] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:41:11.372Z I CONTROL [conn215908] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:41:31.355Z I - [conn205424] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:41:31.364Z I CONTROL [conn205424] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:41:51.361Z I - [conn205487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:41:51.370Z I CONTROL [conn205487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:42:11.346Z I - [conn216144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:42:11.355Z I CONTROL [conn216144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:42:31.364Z I - [conn216136] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:42:31.374Z I CONTROL [conn216136] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:42:51.358Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:42:51.368Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:43:11.357Z I - [conn205517] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:43:11.366Z I CONTROL [conn205517] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:43:31.344Z I - [conn215976] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:43:31.354Z I CONTROL [conn215976] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:43:51.359Z I - [conn207414] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:43:51.369Z I CONTROL [conn207414] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:44:11.349Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:44:11.359Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:44:31.353Z I - [conn205876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:44:31.362Z I CONTROL [conn205876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:44:49.046Z I INDEX [conn215568] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T05:44:51.357Z I - [conn205843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:44:51.367Z I CONTROL [conn205843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:45:11.357Z I - [conn205424] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:45:11.366Z I CONTROL [conn205424] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:45:31.359Z I - [conn215755] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:45:31.369Z I CONTROL [conn215755] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:45:51.355Z I - [conn206329] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:45:51.364Z I CONTROL [conn206329] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:46:11.359Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:46:11.369Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:46:31.347Z I - [conn206771] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:46:31.357Z I CONTROL [conn206771] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:46:51.357Z I - [conn205736] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:46:51.366Z I CONTROL [conn205736] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:47:11.356Z I - [conn205584] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:47:11.366Z I CONTROL [conn205584] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:47:31.352Z I - [conn205584] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:47:31.362Z I CONTROL [conn205584] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:47:51.362Z I - [conn216132] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:47:51.372Z I CONTROL [conn216132] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:48:11.368Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:48:11.378Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:48:31.351Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:48:31.361Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:48:51.354Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:48:51.364Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:49:11.352Z I - [conn216146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:49:11.361Z I CONTROL [conn216146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:49:31.351Z I - [conn216146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:49:31.361Z I CONTROL [conn216146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:49:51.347Z I - [conn205517] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:49:51.356Z I CONTROL [conn205517] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:50:11.344Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:50:11.354Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:50:31.355Z I - [conn205595] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:50:31.365Z I CONTROL [conn205595] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:50:51.355Z I - [conn206774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:50:51.365Z I CONTROL [conn206774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:51:11.357Z I - [conn205572] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:51:11.367Z I CONTROL [conn205572] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:51:31.358Z I - [conn215926] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:51:31.367Z I CONTROL [conn215926] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:51:51.350Z I - [conn216052] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:51:51.360Z I CONTROL [conn216052] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:52:11.343Z I - [conn207414] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:52:11.353Z I CONTROL [conn207414] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:52:31.354Z I - [conn206794] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:52:31.364Z I CONTROL [conn206794] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:52:51.343Z I - [conn215755] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:52:51.352Z I CONTROL [conn215755] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:53:11.339Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:53:11.348Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:53:31.361Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:53:31.370Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:53:51.348Z I - [conn205482] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:53:51.358Z I CONTROL [conn205482] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:54:11.344Z I - [conn215968] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:54:11.353Z I CONTROL [conn215968] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:54:31.345Z I - [conn205752] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:54:31.355Z I CONTROL [conn205752] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:54:51.339Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:54:51.349Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:55:11.342Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:55:11.352Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:55:23.104Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:55:23.114Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:55:31.355Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:55:31.366Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:55:51.351Z I - [conn183336] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:55:51.360Z I CONTROL [conn183336] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:56:11.344Z I - [conn206318] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:56:11.354Z I CONTROL [conn206318] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:56:31.351Z I - [conn205664] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:56:31.361Z I CONTROL [conn205664] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:56:51.362Z I - [conn207413] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:56:51.372Z I CONTROL [conn207413] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:57:11.352Z I - [conn205942] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:57:11.362Z I CONTROL [conn205942] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:57:31.356Z I - [conn178530] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:57:31.366Z I CONTROL [conn178530] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:57:51.356Z I - [conn206313] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:57:51.366Z I CONTROL [conn206313] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:58:11.349Z I - [conn205876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:58:11.359Z I CONTROL [conn205876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:58:31.346Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:58:31.355Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:58:51.348Z I - [conn205587] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:58:51.357Z I CONTROL [conn205587] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:59:11.352Z I - [conn206811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:59:11.361Z I CONTROL [conn206811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:59:31.339Z I - [conn205675] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:59:31.348Z I CONTROL [conn205675] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T05:59:49.133Z I INDEX [conn215461] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T05:59:51.348Z I - [conn207413] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T05:59:51.358Z I CONTROL [conn207413] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:00:11.345Z I - [conn216102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:00:11.354Z I CONTROL [conn216102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:00:31.350Z I - [conn205664] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:00:31.360Z I CONTROL [conn205664] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:00:51.340Z I - [conn177419] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:00:51.349Z I CONTROL [conn177419] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:01:11.352Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:01:11.362Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:01:31.358Z I - [conn216102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:01:31.367Z I CONTROL [conn216102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:01:51.347Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:01:51.356Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:02:11.350Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:02:11.360Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:02:31.358Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:02:31.368Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:02:51.346Z I - [conn205783] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:02:51.356Z I CONTROL [conn205783] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:03:11.338Z I - [conn216052] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:03:11.348Z I CONTROL [conn216052] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:03:31.342Z I - [conn205810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:03:31.351Z I CONTROL [conn205810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:03:51.349Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:03:51.358Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:04:11.360Z I - [conn207414] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:04:11.370Z I CONTROL [conn207414] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:04:31.348Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:04:31.358Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:04:51.349Z I - [conn205749] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:04:51.358Z I CONTROL [conn205749] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:05:11.347Z I - [conn207402] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:05:11.357Z I CONTROL [conn207402] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:05:31.336Z I - [conn205568] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:05:31.346Z I CONTROL [conn205568] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:05:51.345Z I - [conn205937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:05:51.355Z I CONTROL [conn205937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:06:11.338Z I - [conn216145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:06:11.348Z I CONTROL [conn216145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:06:31.345Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:06:31.355Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:06:51.343Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:06:51.353Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:07:11.346Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:07:11.356Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:07:31.347Z I - [conn170471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:07:31.357Z I CONTROL [conn170471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:07:51.347Z I - [conn206257] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:07:51.357Z I CONTROL [conn206257] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:08:11.347Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:08:11.357Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:08:31.335Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:08:31.345Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:08:51.344Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:08:51.354Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:09:11.343Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:09:11.353Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:09:31.347Z I - [conn206811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:09:31.357Z I CONTROL [conn206811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:09:44.775Z I - [conn205554] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:09:44.785Z I CONTROL [conn205554] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:09:51.363Z I - [conn206311] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:09:51.373Z I CONTROL [conn206311] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:10:11.345Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:10:11.354Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:10:31.343Z I - [conn215965] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:10:31.352Z I CONTROL [conn215965] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:10:51.358Z I - [conn178530] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:10:51.367Z I CONTROL [conn178530] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:11:11.343Z I - [conn205656] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:11:11.352Z I CONTROL [conn205656] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:11:31.338Z I - [conn178530] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:11:31.347Z I CONTROL [conn178530] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:11:51.344Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:11:51.354Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:12:11.344Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:12:11.354Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:12:31.363Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:12:31.372Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:12:51.353Z I - [conn205777] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:12:51.363Z I CONTROL [conn205777] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:13:11.343Z I - [conn205949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:13:11.353Z I CONTROL [conn205949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:13:31.341Z I - [conn215953] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:13:31.350Z I CONTROL [conn215953] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:13:51.334Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:13:51.343Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:14:11.333Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:14:11.343Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:14:31.341Z I - [conn205918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:14:31.351Z I CONTROL [conn205918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:14:51.335Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:14:51.345Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:15:11.343Z I - [conn177379] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:15:11.352Z I CONTROL [conn177379] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:15:31.340Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:15:31.350Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:15:51.358Z I - [conn205638] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:15:51.368Z I CONTROL [conn205638] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:16:11.340Z I - [conn205822] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:16:11.350Z I CONTROL [conn205822] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:16:31.341Z I - [conn216131] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:16:31.351Z I CONTROL [conn216131] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:16:51.328Z I - [conn216112] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:16:51.338Z I CONTROL [conn216112] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:17:11.339Z I - [conn205675] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:17:11.349Z I CONTROL [conn205675] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:17:31.343Z I - [conn205948] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:17:31.353Z I CONTROL [conn205948] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:17:51.338Z I - [conn215875] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:17:51.347Z I CONTROL [conn215875] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:18:11.340Z I - [conn205948] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:18:11.349Z I CONTROL [conn205948] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:18:31.341Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:18:31.350Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:18:51.340Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:18:51.349Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:19:11.339Z I - [conn205644] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:19:11.349Z I CONTROL [conn205644] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:19:31.336Z I - [conn216089] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:19:31.346Z I CONTROL [conn216089] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:19:51.338Z I - [conn170487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:19:51.348Z I CONTROL [conn170487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:20:11.336Z I - [conn205675] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:20:11.346Z I CONTROL [conn205675] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:20:31.335Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:20:31.344Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:20:51.346Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:20:51.356Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:21:11.328Z I - [conn205586] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:21:11.338Z I CONTROL [conn205586] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:21:31.338Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:21:31.348Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:21:51.337Z I - [conn205796] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:21:51.346Z I CONTROL [conn205796] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:22:11.347Z I - [conn183343] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:22:11.356Z I CONTROL [conn183343] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:22:31.347Z I - [conn205568] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:22:31.356Z I CONTROL [conn205568] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:22:51.326Z I - [conn206800] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:22:51.336Z I CONTROL [conn206800] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:23:11.335Z I - [conn216081] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:23:11.345Z I CONTROL [conn216081] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:23:31.341Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:23:31.351Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:23:51.326Z I - [conn205933] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:23:51.336Z I CONTROL [conn205933] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:24:11.338Z I - [conn216104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:24:11.348Z I CONTROL [conn216104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:24:31.328Z I - [conn205752] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:24:31.337Z I CONTROL [conn205752] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:24:51.339Z I - [conn205644] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:24:51.349Z I CONTROL [conn205644] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:25:11.335Z I - [conn205863] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:25:11.345Z I CONTROL [conn205863] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:25:31.334Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:25:31.343Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:25:51.337Z I - [conn215850] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:25:51.347Z I CONTROL [conn215850] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:26:11.336Z I - [conn205777] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:26:11.345Z I CONTROL [conn205777] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:26:31.334Z I - [conn215929] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:26:31.344Z I CONTROL [conn215929] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:26:51.335Z I - [conn205482] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:26:51.345Z I CONTROL [conn205482] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:27:11.346Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:27:11.355Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:27:31.335Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:27:31.345Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:27:51.329Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:27:51.338Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:28:11.334Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:28:11.344Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:28:31.331Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:28:31.341Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:28:51.332Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:28:51.342Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:29:11.334Z I - [conn215947] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:29:11.343Z I CONTROL [conn215947] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:29:31.334Z I - [conn215947] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:29:31.344Z I CONTROL [conn215947] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:29:51.325Z I - [conn205945] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:29:51.334Z I CONTROL [conn205945] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:30:11.335Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:30:11.346Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:30:31.330Z I - [conn205804] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:30:31.340Z I CONTROL [conn205804] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:30:51.335Z I - [conn215913] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:30:51.344Z I CONTROL [conn215913] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:31:11.336Z I - [conn170487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:31:11.346Z I CONTROL [conn170487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:31:31.332Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:31:31.342Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:31:51.333Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:31:51.343Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:32:11.330Z I - [conn205570] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:32:11.340Z I CONTROL [conn205570] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:32:31.341Z I - [conn205801] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:32:31.351Z I CONTROL [conn205801] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:32:51.323Z I - [conn183103] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:32:51.332Z I CONTROL [conn183103] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:33:11.330Z I - [conn183103] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:33:11.339Z I CONTROL [conn183103] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:33:31.342Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:33:31.351Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:33:51.322Z I - [conn206813] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:33:51.332Z I CONTROL [conn206813] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:34:11.328Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:34:11.337Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:34:31.330Z I - [conn205876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:34:31.340Z I CONTROL [conn205876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:34:51.331Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:34:51.340Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:35:11.329Z I - [conn215947] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:35:11.338Z I CONTROL [conn215947] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:35:31.337Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:35:31.347Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:35:51.326Z I - [conn205787] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:35:51.336Z I CONTROL [conn205787] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:36:11.339Z I - [conn205737] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:36:11.349Z I CONTROL [conn205737] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:36:29.100Z I - [conn216095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:36:29.111Z I CONTROL [conn216095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:36:31.344Z I - [conn216095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:36:31.354Z I CONTROL [conn216095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:36:51.321Z I - [conn205949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:36:51.330Z I CONTROL [conn205949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:37:11.338Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:37:11.347Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:37:31.320Z I - [conn205742] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:37:31.330Z I CONTROL [conn205742] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:37:46.469Z I - [conn216147] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:37:46.478Z I CONTROL [conn216147] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:37:51.318Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:37:51.328Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:38:11.324Z I - [conn212230] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:38:11.333Z I CONTROL [conn212230] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:38:31.320Z I - [conn212230] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:38:31.330Z I CONTROL [conn212230] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:38:51.324Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:38:51.334Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:39:11.320Z I - [conn212229] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:39:11.330Z I CONTROL [conn212229] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:39:31.328Z I - [conn205656] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:39:31.337Z I CONTROL [conn205656] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:39:51.329Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:39:51.339Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:40:11.327Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:40:11.337Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:40:31.338Z I - [conn205626] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:40:31.348Z I CONTROL [conn205626] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:40:51.318Z I - [conn205930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:40:51.328Z I CONTROL [conn205930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:41:11.334Z I - [conn216097] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:41:11.344Z I CONTROL [conn216097] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:41:31.328Z I - [conn205749] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:41:31.338Z I CONTROL [conn205749] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:41:51.331Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:41:51.340Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:42:11.341Z I - [conn205930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:42:11.351Z I CONTROL [conn205930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:42:31.328Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:42:31.338Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:42:51.324Z I - [conn215874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:42:51.333Z I CONTROL [conn215874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:43:11.329Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:43:11.339Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:43:31.335Z I - [conn216129] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:43:31.344Z I CONTROL [conn216129] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:43:51.344Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:43:51.354Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:44:11.321Z I - [conn215910] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:44:11.330Z I CONTROL [conn215910] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:44:31.327Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:44:31.337Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:44:51.324Z I - [conn215926] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:44:51.334Z I CONTROL [conn215926] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:45:11.327Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:45:11.337Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:45:22.344Z I - [conn212225] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:45:22.354Z I CONTROL [conn212225] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:45:31.327Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:45:31.337Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:45:51.317Z I - [conn215143] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:45:51.327Z I CONTROL [conn215143] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:46:11.330Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:46:11.340Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:46:31.328Z I - [conn216095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:46:31.337Z I CONTROL [conn216095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:46:51.324Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:46:51.333Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:47:11.317Z I - [conn205918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:47:11.327Z I CONTROL [conn205918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:47:20.662Z I - [conn216112] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:47:20.673Z I CONTROL [conn216112] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:47:31.330Z I - [conn206800] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:47:31.340Z I CONTROL [conn206800] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:47:51.325Z I - [conn205829] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:47:51.335Z I CONTROL [conn205829] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:48:11.324Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:48:11.334Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:48:31.321Z I - [conn205874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:48:31.330Z I CONTROL [conn205874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:48:51.334Z I - [conn170812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:48:51.344Z I CONTROL [conn170812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:49:11.325Z I - [conn170473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:49:11.334Z I CONTROL [conn170473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:49:31.318Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:49:31.327Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:49:51.316Z I - [conn207412] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:49:51.325Z I CONTROL [conn207412] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:50:11.327Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:50:11.338Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:50:31.333Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:50:31.343Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:50:51.326Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:50:51.336Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:51:11.326Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:51:11.336Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:51:31.333Z I - [conn205843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:51:31.342Z I CONTROL [conn205843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:51:51.320Z I - [conn212226] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:51:51.330Z I CONTROL [conn212226] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:52:11.325Z I - [conn207405] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:52:11.335Z I CONTROL [conn207405] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:52:31.320Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:52:31.329Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:52:51.319Z I - [conn216095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:52:51.329Z I CONTROL [conn216095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:53:11.313Z I - [conn215968] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:53:11.323Z I CONTROL [conn215968] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:53:31.324Z I - [conn207413] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:53:31.334Z I CONTROL [conn207413] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:53:51.322Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:53:51.332Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:54:11.313Z I - [conn205863] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:54:11.322Z I CONTROL [conn205863] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:54:31.312Z I - [conn205874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:54:31.323Z I CONTROL [conn205874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:54:51.321Z I - [conn206320] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:54:51.330Z I CONTROL [conn206320] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:55:11.311Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:55:11.320Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:55:31.320Z I - [conn206330] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:55:31.329Z I CONTROL [conn206330] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:55:51.319Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:55:51.328Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:56:11.314Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:56:11.324Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:56:31.319Z I - [conn205941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:56:31.328Z I CONTROL [conn205941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:56:51.311Z I - [conn205876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:56:51.321Z I CONTROL [conn205876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:57:11.310Z I - [conn216112] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:57:11.319Z I CONTROL [conn216112] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:57:31.329Z I - [conn216136] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:57:31.338Z I CONTROL [conn216136] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:57:51.330Z I - [conn205749] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:57:51.340Z I CONTROL [conn205749] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:58:11.315Z I - [conn216146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:58:11.325Z I CONTROL [conn216146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:58:31.321Z I - [conn205796] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:58:31.331Z I CONTROL [conn205796] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:58:51.319Z I - [conn216110] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:58:51.329Z I CONTROL [conn216110] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:59:11.310Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:59:11.320Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:59:31.325Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:59:31.334Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T06:59:49.885Z I INDEX [conn215461] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T06:59:51.318Z I - [conn205586] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T06:59:51.327Z I CONTROL [conn205586] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:00:11.323Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:00:11.332Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:00:31.322Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:00:31.332Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:00:51.320Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:00:51.329Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:01:11.324Z I - [conn205644] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:01:11.333Z I CONTROL [conn205644] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:01:31.310Z I - [conn205743] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:01:31.320Z I CONTROL [conn205743] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:01:51.318Z I - [conn216069] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:01:51.328Z I CONTROL [conn216069] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:02:11.324Z I - [conn215956] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:02:11.333Z I CONTROL [conn215956] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:02:31.307Z I - [conn216107] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:02:31.317Z I CONTROL [conn216107] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:02:51.328Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:02:51.337Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:03:11.326Z I - [conn215875] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:03:11.335Z I CONTROL [conn215875] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:03:31.308Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:03:31.317Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:03:51.317Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:03:51.327Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:04:11.331Z I - [conn205456] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:04:11.341Z I CONTROL [conn205456] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:04:31.333Z I - [conn206775] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:04:31.342Z I CONTROL [conn206775] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:04:51.320Z I - [conn170487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:04:51.330Z I CONTROL [conn170487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:05:11.328Z I - [conn215953] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:05:11.337Z I CONTROL [conn215953] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:05:31.318Z I - [conn205500] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:05:31.328Z I CONTROL [conn205500] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:05:51.313Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:05:51.322Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:06:11.306Z I - [conn216153] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:06:11.316Z I CONTROL [conn216153] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:06:31.305Z I - [conn216153] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:06:31.314Z I CONTROL [conn216153] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:06:51.306Z I - [conn216116] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:06:51.315Z I CONTROL [conn216116] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:07:11.307Z I - [conn205752] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:07:11.317Z I CONTROL [conn205752] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:07:31.304Z I - [conn206328] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:07:31.314Z I CONTROL [conn206328] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:07:51.316Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:07:51.326Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:08:11.311Z I - [conn206314] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:08:11.321Z I CONTROL [conn206314] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:08:31.317Z I - [conn216144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:08:31.326Z I CONTROL [conn216144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:08:51.313Z I - [conn215966] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:08:51.323Z I CONTROL [conn215966] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:09:11.315Z I - [conn205810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:09:11.325Z I CONTROL [conn205810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:09:31.306Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:09:31.315Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:09:51.319Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:09:51.330Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:10:11.317Z I - [conn205815] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:10:11.327Z I CONTROL [conn205815] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:10:31.322Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:10:31.332Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:10:51.303Z I - [conn216104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:10:51.313Z I CONTROL [conn216104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:11:11.314Z I - [conn206298] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:11:11.324Z I CONTROL [conn206298] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:11:31.314Z I - [conn212600] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:11:31.323Z I CONTROL [conn212600] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:11:51.311Z I - [conn205742] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:11:51.321Z I CONTROL [conn205742] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:12:11.311Z I - [conn205726] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:12:11.321Z I CONTROL [conn205726] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:12:31.312Z I - [conn216124] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:12:31.322Z I CONTROL [conn216124] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:12:51.315Z I - [conn216069] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:12:51.325Z I CONTROL [conn216069] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:13:11.302Z I - [conn215962] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:13:11.311Z I CONTROL [conn215962] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:13:31.314Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:13:31.323Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:13:51.308Z I - [conn205822] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:13:51.318Z I CONTROL [conn205822] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:14:11.313Z I - [conn215975] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:14:11.322Z I CONTROL [conn215975] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:14:31.312Z I - [conn205499] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:14:31.322Z I CONTROL [conn205499] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:14:51.301Z I - [conn215953] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:14:51.310Z I CONTROL [conn215953] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:15:11.312Z I - [conn205787] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:15:11.322Z I CONTROL [conn205787] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:15:31.319Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:15:31.329Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:15:51.305Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:15:51.314Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:16:11.303Z I - [conn205598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:16:11.312Z I CONTROL [conn205598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:16:31.308Z I - [conn215144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:16:31.317Z I CONTROL [conn215144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:16:51.312Z I - [conn215926] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:16:51.321Z I CONTROL [conn215926] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:17:11.312Z I - [conn205726] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:17:11.322Z I CONTROL [conn205726] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:17:31.311Z I - [conn205737] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:17:31.321Z I CONTROL [conn205737] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:17:51.315Z I - [conn205814] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:17:51.325Z I CONTROL [conn205814] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:18:11.308Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:18:11.318Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:18:31.323Z I - [conn205570] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:18:31.333Z I CONTROL [conn205570] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:18:51.312Z I - [conn205587] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:18:51.322Z I CONTROL [conn205587] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:19:11.302Z I - [conn178530] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:19:11.312Z I CONTROL [conn178530] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:19:31.302Z I - [conn205570] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:19:31.311Z I CONTROL [conn205570] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:19:51.311Z I - [conn216145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:19:51.321Z I CONTROL [conn216145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:20:11.313Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:20:11.323Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:20:31.300Z I - [conn177419] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:20:31.309Z I CONTROL [conn177419] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:20:51.297Z I - [conn215929] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:20:51.307Z I CONTROL [conn215929] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:21:11.300Z I - [conn215976] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:21:11.309Z I CONTROL [conn215976] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:21:31.309Z I - [conn205471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:21:31.318Z I CONTROL [conn205471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:21:51.304Z I - [conn215855] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:21:51.313Z I CONTROL [conn215855] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:22:11.300Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:22:11.309Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:22:31.307Z I - [conn205664] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:22:31.316Z I CONTROL [conn205664] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:22:51.308Z I - [conn206321] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:22:51.318Z I CONTROL [conn206321] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:23:11.302Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:23:11.312Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:23:31.318Z I - [conn205933] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:23:31.328Z I CONTROL [conn205933] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:23:51.319Z I - [conn205933] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:23:51.328Z I CONTROL [conn205933] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:24:11.311Z I - [conn215910] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:24:11.321Z I CONTROL [conn215910] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:24:31.307Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:24:31.317Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:24:51.305Z I - [conn216112] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:24:51.314Z I CONTROL [conn216112] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:25:11.306Z I - [conn205791] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:25:11.316Z I CONTROL [conn205791] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:25:17.846Z I - [conn212230] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:25:17.856Z I CONTROL [conn212230] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:25:31.304Z I - [conn183171] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:25:31.313Z I CONTROL [conn183171] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:25:51.305Z I - [conn215926] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:25:51.314Z I CONTROL [conn215926] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:26:11.321Z I - [conn215947] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:26:11.330Z I CONTROL [conn215947] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:26:31.311Z I - [conn207412] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:26:31.320Z I CONTROL [conn207412] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:26:51.313Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:26:51.322Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:27:11.309Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:27:11.318Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:27:31.308Z I - [conn170471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:27:31.318Z I CONTROL [conn170471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:27:51.307Z I - [conn205456] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:27:51.317Z I CONTROL [conn205456] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:28:11.293Z I - [conn215968] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:28:11.303Z I CONTROL [conn215968] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:28:31.302Z I - [conn215874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:28:31.311Z I CONTROL [conn215874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:28:51.301Z I - [conn216127] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:28:51.311Z I CONTROL [conn216127] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:29:11.301Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:29:11.311Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:29:31.305Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:29:31.314Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:29:51.307Z I - [conn205634] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:29:51.317Z I CONTROL [conn205634] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:30:11.304Z I - [conn216122] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:30:11.313Z I CONTROL [conn216122] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:30:31.311Z I - [conn205822] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:30:31.320Z I CONTROL [conn205822] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:30:51.295Z I - [conn205783] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:30:51.305Z I CONTROL [conn205783] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:31:11.305Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:31:11.315Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:31:31.303Z I - [conn215939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:31:31.313Z I CONTROL [conn215939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:31:51.303Z I - [conn216069] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:31:51.312Z I CONTROL [conn216069] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:32:11.303Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:32:11.313Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:32:31.306Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:32:31.315Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:32:51.311Z I - [conn216069] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:32:51.320Z I CONTROL [conn216069] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:33:11.302Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:33:11.311Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:33:31.300Z I - [conn215140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:33:31.309Z I CONTROL [conn215140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:33:51.302Z I - [conn205656] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:33:51.312Z I CONTROL [conn205656] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:34:11.289Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:34:11.299Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:34:31.302Z I - [conn215854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:34:31.312Z I CONTROL [conn215854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:34:51.299Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:34:51.309Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:35:11.294Z I - [conn205872] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:35:11.303Z I CONTROL [conn205872] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:35:31.302Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:35:31.312Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:35:51.303Z I - [conn212226] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:35:51.313Z I CONTROL [conn212226] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:36:11.295Z I - [conn215947] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:36:11.305Z I CONTROL [conn215947] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:36:31.298Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:36:31.307Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:36:51.300Z I - [conn205868] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:36:51.309Z I CONTROL [conn205868] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:37:11.292Z I - [conn205791] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:37:11.302Z I CONTROL [conn205791] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:37:31.303Z I - [conn207412] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:37:31.313Z I CONTROL [conn207412] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:37:51.300Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:37:51.309Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:38:11.302Z I - [conn205595] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:38:11.311Z I CONTROL [conn205595] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:38:31.289Z I - [conn215143] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:38:31.299Z I CONTROL [conn215143] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:38:51.288Z I - [conn215950] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:38:51.298Z I CONTROL [conn215950] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:39:11.301Z I - [conn215144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:39:11.311Z I CONTROL [conn215144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:39:31.298Z I - [conn205829] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:39:31.307Z I CONTROL [conn205829] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:39:51.298Z I - [conn216156] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:39:51.307Z I CONTROL [conn216156] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:40:11.289Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:40:11.299Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:40:31.298Z I - [conn205572] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:40:31.307Z I CONTROL [conn205572] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:40:51.302Z I - [conn206298] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:40:51.312Z I CONTROL [conn206298] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:41:11.289Z I - [conn212230] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:41:11.299Z I CONTROL [conn212230] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:41:31.302Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:41:31.312Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:41:51.299Z I - [conn215850] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:41:51.309Z I CONTROL [conn215850] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:42:11.309Z I - [conn206771] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:42:11.319Z I CONTROL [conn206771] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:42:31.298Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:42:31.307Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:42:51.288Z I - [conn205500] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:42:51.298Z I CONTROL [conn205500] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:43:11.301Z I - [conn205801] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:43:11.311Z I CONTROL [conn205801] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:43:31.303Z I - [conn216136] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:43:31.312Z I CONTROL [conn216136] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:43:51.313Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:43:51.323Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:44:11.294Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:44:11.304Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:44:31.299Z I - [conn205872] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:44:31.308Z I CONTROL [conn205872] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:44:51.310Z I - [conn207408] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:44:51.320Z I CONTROL [conn207408] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:45:11.290Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:45:11.300Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:45:31.308Z I - [conn205526] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:45:31.317Z I CONTROL [conn205526] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:45:51.292Z I - [conn215911] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:45:51.302Z I CONTROL [conn215911] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:46:11.289Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:46:11.298Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:46:31.284Z I - [conn215858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:46:31.294Z I CONTROL [conn215858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:46:51.287Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:46:51.297Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:47:11.286Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:47:11.295Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:47:31.296Z I - [conn212600] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:47:31.305Z I CONTROL [conn212600] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:47:51.285Z I - [conn215850] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:47:51.295Z I CONTROL [conn215850] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:48:11.296Z I - [conn216132] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:48:11.305Z I CONTROL [conn216132] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:48:31.287Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:48:31.297Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:48:51.295Z I - [conn205626] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:48:51.306Z I CONTROL [conn205626] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:49:11.303Z I - [conn205615] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:49:11.313Z I CONTROL [conn205615] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:49:31.297Z I - [conn205570] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:49:31.307Z I CONTROL [conn205570] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:49:51.294Z I - [conn215976] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:49:51.304Z I CONTROL [conn215976] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:50:11.296Z I - [conn205587] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:50:11.305Z I CONTROL [conn205587] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:50:31.294Z I - [conn216145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:50:31.303Z I CONTROL [conn216145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:50:51.286Z I - [conn183103] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:50:51.295Z I CONTROL [conn183103] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:51:11.294Z I - [conn215962] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:51:11.303Z I CONTROL [conn215962] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:51:31.286Z I - [conn205777] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:51:31.295Z I CONTROL [conn205777] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:51:51.296Z I - [conn206775] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:51:51.306Z I CONTROL [conn206775] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:52:11.289Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:52:11.299Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:52:31.284Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:52:31.294Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:52:51.297Z I - [conn205656] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:52:51.306Z I CONTROL [conn205656] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:53:11.297Z I - [conn205726] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:53:11.306Z I CONTROL [conn205726] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:53:31.293Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:53:31.302Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:53:51.293Z I - [conn205856] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:53:51.304Z I CONTROL [conn205856] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:54:11.284Z I - [conn205868] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:54:11.293Z I CONTROL [conn205868] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:54:31.283Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:54:31.293Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:54:51.291Z I - [conn215919] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:54:51.301Z I CONTROL [conn215919] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:55:11.295Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:55:11.304Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:55:31.296Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:55:31.306Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:55:51.293Z I - [conn215983] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:55:51.302Z I CONTROL [conn215983] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:56:11.296Z I - [conn215983] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:56:11.306Z I CONTROL [conn215983] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:56:31.294Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:56:31.303Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:56:51.295Z I - [conn205941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:56:51.304Z I CONTROL [conn205941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:57:11.290Z I - [conn215849] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:57:11.299Z I CONTROL [conn215849] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:57:31.295Z I - [conn205856] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:57:31.304Z I CONTROL [conn205856] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:57:51.284Z I - [conn216156] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:57:51.293Z I CONTROL [conn216156] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:58:11.282Z I - [conn205570] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:58:11.293Z I CONTROL [conn205570] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:58:31.290Z I - [conn216119] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:58:31.300Z I CONTROL [conn216119] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:58:51.280Z I - [conn215926] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:58:51.290Z I CONTROL [conn215926] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:59:11.290Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:59:11.300Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:59:31.292Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:59:31.303Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T07:59:49.458Z I INDEX [conn215461] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T07:59:51.292Z I - [conn205858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T07:59:51.302Z I CONTROL [conn205858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:00:11.290Z I - [conn216145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:00:11.300Z I CONTROL [conn216145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:00:31.283Z I - [conn215977] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:00:31.292Z I CONTROL [conn215977] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:00:51.297Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:00:51.307Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:01:11.284Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:01:11.293Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:01:31.282Z I - [conn205471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:01:31.291Z I CONTROL [conn205471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:01:51.292Z I - [conn205499] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:01:51.301Z I CONTROL [conn205499] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:02:11.291Z I - [conn207405] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:02:11.300Z I CONTROL [conn207405] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:02:31.293Z I - [conn205944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:02:31.303Z I CONTROL [conn205944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:02:51.288Z I - [conn216116] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:02:51.297Z I CONTROL [conn216116] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:03:11.293Z I - [conn177379] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:03:11.303Z I CONTROL [conn177379] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:03:31.289Z I - [conn177379] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:03:31.299Z I CONTROL [conn177379] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:03:51.290Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:03:51.300Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:04:11.294Z I - [conn205453] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:04:11.303Z I CONTROL [conn205453] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:04:31.286Z I - [conn216127] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:04:31.295Z I CONTROL [conn216127] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:04:51.288Z I - [conn215931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:04:51.298Z I CONTROL [conn215931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:05:11.286Z I - [conn205500] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:05:11.296Z I CONTROL [conn205500] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:05:31.286Z I - [conn215931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:05:31.295Z I CONTROL [conn215931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:05:51.288Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:05:51.298Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:06:11.295Z I - [conn183343] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:06:11.305Z I CONTROL [conn183343] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:06:31.294Z I - [conn205863] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:06:31.304Z I CONTROL [conn205863] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:06:51.288Z I - [conn206332] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:06:51.298Z I CONTROL [conn206332] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:07:11.277Z I - [conn215957] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:07:11.287Z I CONTROL [conn215957] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:07:31.278Z I - [conn216124] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:07:31.287Z I CONTROL [conn216124] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:07:51.293Z I - [conn205930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:07:51.303Z I CONTROL [conn205930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:08:11.286Z I - [conn205882] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:08:11.295Z I CONTROL [conn205882] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:08:31.281Z I - [conn216071] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:08:31.291Z I CONTROL [conn216071] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:08:51.278Z I - [conn205853] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:08:51.288Z I CONTROL [conn205853] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:09:11.278Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:09:11.287Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:09:31.273Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:09:31.283Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:09:51.275Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:09:51.285Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:09:57.746Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:09:57.756Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:10:00.302Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:10:00.312Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:10:11.287Z I - [conn206316] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:10:11.296Z I CONTROL [conn206316] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:10:31.278Z I - [conn206807] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:10:31.288Z I CONTROL [conn206807] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:10:51.281Z I - [conn205857] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:10:51.290Z I CONTROL [conn205857] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:11:11.286Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:11:11.295Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:11:31.278Z I - [conn206800] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:11:31.287Z I CONTROL [conn206800] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:11:51.288Z I - [conn216129] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:11:51.298Z I CONTROL [conn216129] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:12:11.283Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:12:11.292Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:12:31.279Z I - [conn205872] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:12:31.288Z I CONTROL [conn205872] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:12:51.287Z I - [conn216131] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:12:51.296Z I CONTROL [conn216131] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:13:11.286Z I - [conn205587] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:13:11.295Z I CONTROL [conn205587] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:13:31.283Z I - [conn206318] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:13:31.293Z I CONTROL [conn206318] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:13:51.285Z I - [conn205726] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:13:51.295Z I CONTROL [conn205726] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:14:11.285Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:14:11.295Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:14:31.282Z I - [conn216131] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:14:31.292Z I CONTROL [conn216131] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:14:51.289Z I - [conn205743] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:14:51.299Z I CONTROL [conn205743] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:15:11.286Z I - [conn215918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:15:11.295Z I CONTROL [conn215918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:15:31.281Z I - [conn205900] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:15:31.291Z I CONTROL [conn205900] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:15:51.284Z I - [conn183429] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:15:51.294Z I CONTROL [conn183429] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:16:11.282Z I - [conn205777] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:16:11.292Z I CONTROL [conn205777] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:16:31.273Z I - [conn215918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:16:31.283Z I CONTROL [conn215918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:16:51.284Z I - [conn205691] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:16:51.294Z I CONTROL [conn205691] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:17:11.272Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:17:11.282Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:17:31.285Z I - [conn206774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:17:31.294Z I CONTROL [conn206774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:17:51.280Z I - [conn216108] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:17:51.289Z I CONTROL [conn216108] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:18:11.280Z I - [conn216069] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:18:11.290Z I CONTROL [conn216069] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:18:31.280Z I - [conn205760] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:18:31.290Z I CONTROL [conn205760] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:18:51.277Z I - [conn205940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:18:51.287Z I CONTROL [conn205940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:19:11.282Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:19:11.292Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:19:31.276Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:19:31.285Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:19:51.284Z I - [conn216143] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:19:51.294Z I CONTROL [conn216143] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:20:11.271Z I - [conn215968] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:20:11.280Z I CONTROL [conn215968] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:20:31.273Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:20:31.283Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:20:51.284Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:20:51.294Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:21:11.275Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:21:11.284Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:21:31.273Z I - [conn205568] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:21:31.282Z I CONTROL [conn205568] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:21:51.279Z I - [conn216124] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:21:51.288Z I CONTROL [conn216124] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:22:11.274Z I - [conn205908] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:22:11.284Z I CONTROL [conn205908] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:22:31.287Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:22:31.296Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:22:51.280Z I - [conn205941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:22:51.290Z I CONTROL [conn205941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:23:11.278Z I - [conn215910] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:23:11.287Z I CONTROL [conn215910] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:23:31.271Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:23:31.281Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:23:51.279Z I - [conn215854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:23:51.288Z I CONTROL [conn215854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:24:11.286Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:24:11.295Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:24:31.279Z I - [conn170471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:24:31.289Z I CONTROL [conn170471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:24:51.291Z I - [conn205681] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:24:51.300Z I CONTROL [conn205681] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:25:11.325Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:25:11.335Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:25:31.278Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:25:31.288Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:25:51.280Z I - [conn205737] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:25:51.289Z I CONTROL [conn205737] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:26:11.290Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:26:11.299Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:26:31.275Z I - [conn215880] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:26:31.285Z I CONTROL [conn215880] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:26:51.271Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:26:51.280Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:27:11.268Z I - [conn205872] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:27:11.277Z I CONTROL [conn205872] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:27:31.278Z I - [conn205874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:27:31.287Z I CONTROL [conn205874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:27:51.276Z I - [conn215854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:27:51.286Z I CONTROL [conn215854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:28:11.279Z I - [conn215979] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:28:11.289Z I CONTROL [conn215979] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:28:31.281Z I - [conn206327] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:28:31.290Z I CONTROL [conn206327] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:28:51.278Z I - [conn215953] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:28:51.287Z I CONTROL [conn215953] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:29:11.279Z I - [conn212230] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:29:11.289Z I CONTROL [conn212230] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:29:31.277Z I - [conn216141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:29:31.287Z I CONTROL [conn216141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:29:51.280Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:29:51.290Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:30:11.274Z I - [conn215927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:30:11.283Z I CONTROL [conn215927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:30:31.275Z I - [conn183336] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:30:31.285Z I CONTROL [conn183336] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:30:51.278Z I - [conn205442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:30:51.288Z I CONTROL [conn205442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:31:11.273Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:31:11.282Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:31:31.278Z I - [conn216080] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:31:31.287Z I CONTROL [conn216080] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:31:51.289Z I - [conn183115] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:31:51.299Z I CONTROL [conn183115] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:32:11.275Z I - [conn206311] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:32:11.284Z I CONTROL [conn206311] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:32:31.270Z I - [conn205940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:32:31.280Z I CONTROL [conn205940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:32:51.277Z I - [conn205749] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:32:51.287Z I CONTROL [conn205749] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:33:11.275Z I - [conn205615] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:33:11.285Z I CONTROL [conn205615] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:33:31.277Z I - [conn212600] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:33:31.287Z I CONTROL [conn212600] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:33:51.278Z I - [conn212600] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:33:51.289Z I CONTROL [conn212600] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:34:11.276Z I - [conn205843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:34:11.286Z I CONTROL [conn205843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:34:31.277Z I - [conn205843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:34:31.287Z I CONTROL [conn205843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:34:51.274Z I - [conn216089] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:34:51.284Z I CONTROL [conn216089] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:35:11.268Z I - [conn206332] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:35:11.278Z I CONTROL [conn206332] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:35:31.274Z I - [conn205453] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:35:31.283Z I CONTROL [conn205453] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:35:51.271Z I - [conn215963] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:35:51.281Z I CONTROL [conn215963] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:36:11.262Z I - [conn215977] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:36:11.271Z I CONTROL [conn215977] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:36:31.263Z I - [conn205944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:36:31.273Z I CONTROL [conn205944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:36:51.281Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:36:51.292Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:37:11.274Z I - [conn205878] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:37:11.284Z I CONTROL [conn205878] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:37:31.275Z I - [conn216135] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:37:31.284Z I CONTROL [conn216135] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:37:51.276Z I - [conn205942] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:37:51.286Z I CONTROL [conn205942] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:38:11.272Z I - [conn216147] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:38:11.282Z I CONTROL [conn216147] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:38:31.274Z I - [conn215910] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:38:31.284Z I CONTROL [conn215910] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:38:51.276Z I - [conn205453] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:38:51.286Z I CONTROL [conn205453] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:39:11.278Z I - [conn215855] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:39:11.288Z I CONTROL [conn215855] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:39:31.275Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:39:31.284Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:39:51.269Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:39:51.280Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:40:11.270Z I - [conn216135] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:40:11.281Z I CONTROL [conn216135] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:40:31.270Z I - [conn215977] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:40:31.280Z I CONTROL [conn215977] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:40:51.270Z I - [conn205868] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:40:51.280Z I CONTROL [conn205868] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:41:11.268Z I - [conn216143] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:41:11.277Z I CONTROL [conn216143] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:41:31.720Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:41:31.730Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:41:51.267Z I - [conn215876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:41:51.277Z I CONTROL [conn215876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:42:11.265Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:42:11.275Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:42:31.269Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:42:31.279Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:42:51.268Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:42:51.277Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:43:11.271Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:43:11.281Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:43:31.269Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:43:31.279Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:43:51.272Z I - [conn207408] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:43:51.282Z I CONTROL [conn207408] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:44:11.267Z I - [conn216125] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:44:11.276Z I CONTROL [conn216125] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:44:31.269Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:44:31.279Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:44:51.271Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:44:51.281Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:45:11.269Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:45:11.278Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:45:31.269Z I - [conn215975] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:45:31.278Z I CONTROL [conn215975] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:45:51.261Z I - [conn205453] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:45:51.271Z I CONTROL [conn205453] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:46:11.267Z I - [conn215859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:46:11.277Z I CONTROL [conn215859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:46:31.270Z I - [conn216144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:46:31.279Z I CONTROL [conn216144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:46:51.268Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:46:51.278Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:47:11.265Z I - [conn216115] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:47:11.274Z I CONTROL [conn216115] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:47:31.270Z I - [conn216052] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:47:31.279Z I CONTROL [conn216052] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:47:51.264Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:47:51.274Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:48:11.261Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:48:11.271Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:48:31.265Z I - [conn216127] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:48:31.274Z I CONTROL [conn216127] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:48:51.266Z I - [conn205473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:48:51.276Z I CONTROL [conn205473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:49:11.268Z I - [conn216123] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:49:11.278Z I CONTROL [conn216123] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:49:31.256Z I - [conn170487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:49:31.266Z I CONTROL [conn170487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:49:51.264Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:49:51.274Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:50:11.266Z I - [conn206329] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:50:11.275Z I CONTROL [conn206329] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:50:31.266Z I - [conn207413] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:50:31.276Z I CONTROL [conn207413] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:50:51.265Z I - [conn216089] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:50:51.274Z I CONTROL [conn216089] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:51:11.262Z I - [conn183104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:51:11.272Z I CONTROL [conn183104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:51:31.265Z I - [conn205675] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:51:31.275Z I CONTROL [conn205675] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:51:51.281Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:51:51.290Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:52:11.265Z I - [conn205935] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:52:11.275Z I CONTROL [conn205935] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:52:22.506Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:52:22.515Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:52:31.267Z I - [conn216113] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:52:31.277Z I CONTROL [conn216113] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:52:51.276Z I - [conn205562] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:52:51.286Z I CONTROL [conn205562] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:53:11.263Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:53:11.273Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:53:31.272Z I - [conn216113] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:53:31.281Z I CONTROL [conn216113] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:53:51.261Z I - [conn216129] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:53:51.271Z I CONTROL [conn216129] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:54:11.257Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:54:11.267Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:54:31.266Z I - [conn205878] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:54:31.276Z I CONTROL [conn205878] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:54:51.254Z I - [conn205697] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:54:51.264Z I CONTROL [conn205697] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:55:11.264Z I - [conn206255] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:55:11.273Z I CONTROL [conn206255] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:55:31.262Z I - [conn206313] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:55:31.272Z I CONTROL [conn206313] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:55:51.264Z I - [conn206313] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:55:51.274Z I CONTROL [conn206313] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:56:11.258Z I - [conn205586] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:56:11.268Z I CONTROL [conn205586] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:56:31.259Z I - [conn215957] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:56:31.268Z I CONTROL [conn215957] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:56:51.257Z I - [conn215962] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:56:51.267Z I CONTROL [conn215962] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:57:11.269Z I - [conn207414] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:57:11.279Z I CONTROL [conn207414] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:57:31.253Z I - [conn205740] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:57:31.262Z I CONTROL [conn205740] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:57:51.261Z I - [conn215910] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:57:51.270Z I CONTROL [conn215910] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:58:11.265Z I - [conn205944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:58:11.275Z I CONTROL [conn205944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:58:31.261Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:58:31.270Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:58:51.268Z I - [conn205949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:58:51.278Z I CONTROL [conn205949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:59:11.250Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:59:11.259Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:59:31.253Z I - [conn205874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:59:31.263Z I CONTROL [conn205874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T08:59:48.935Z I INDEX [conn215461] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T08:59:51.262Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T08:59:51.272Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:00:11.250Z I - [conn216096] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:00:11.260Z I CONTROL [conn216096] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:00:31.268Z I - [conn170471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:00:31.278Z I CONTROL [conn170471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:00:51.261Z I - [conn216132] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:00:51.270Z I CONTROL [conn216132] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:01:11.262Z I - [conn205471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:01:11.272Z I CONTROL [conn205471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:01:31.258Z I - [conn215919] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:01:31.268Z I CONTROL [conn215919] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:01:51.264Z I - [conn205736] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:01:51.274Z I CONTROL [conn205736] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:02:11.255Z I - [conn205815] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:02:11.265Z I CONTROL [conn205815] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:02:31.258Z I - [conn215877] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:02:31.267Z I CONTROL [conn215877] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:02:51.255Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:02:51.265Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:03:11.253Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:03:11.262Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:03:31.250Z I - [conn205944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:03:31.260Z I CONTROL [conn205944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:03:51.258Z I - [conn205908] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:03:51.268Z I CONTROL [conn205908] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:04:11.261Z I - [conn205858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:04:11.271Z I CONTROL [conn205858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:04:31.253Z I - [conn206319] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:04:31.263Z I CONTROL [conn206319] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:04:51.246Z I - [conn216134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:04:51.256Z I CONTROL [conn216134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:05:11.259Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:05:11.268Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:05:31.249Z I - [conn215145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:05:31.259Z I CONTROL [conn215145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:05:51.259Z I - [conn205804] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:05:51.269Z I CONTROL [conn205804] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:06:11.263Z I - [conn206797] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:06:11.273Z I CONTROL [conn206797] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:06:31.260Z I - [conn215922] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:06:31.270Z I CONTROL [conn215922] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:06:51.249Z I - [conn215918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:06:51.259Z I CONTROL [conn215918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:07:11.259Z I - [conn216081] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:07:11.269Z I CONTROL [conn216081] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:07:31.256Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:07:31.266Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:07:51.259Z I - [conn216080] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:07:51.268Z I CONTROL [conn216080] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:08:11.262Z I - [conn205484] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:08:11.272Z I CONTROL [conn205484] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:08:31.257Z I - [conn206316] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:08:31.267Z I CONTROL [conn206316] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:08:51.257Z I - [conn205633] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:08:51.267Z I CONTROL [conn205633] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:09:11.254Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:09:11.264Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:09:31.256Z I - [conn205798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:09:31.266Z I CONTROL [conn205798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:09:51.252Z I - [conn205798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:09:51.262Z I CONTROL [conn205798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:10:11.261Z I - [conn205840] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:10:11.271Z I CONTROL [conn205840] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:10:31.253Z I - [conn215913] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:10:31.263Z I CONTROL [conn215913] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:10:51.254Z I - [conn216052] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:10:51.264Z I CONTROL [conn216052] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:11:11.260Z I - [conn212230] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:11:11.270Z I CONTROL [conn212230] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:11:31.259Z I - [conn216141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:11:31.268Z I CONTROL [conn216141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:11:51.251Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:11:51.261Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:12:11.256Z I - [conn205562] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:12:11.266Z I CONTROL [conn205562] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:12:31.255Z I - [conn215952] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:12:31.265Z I CONTROL [conn215952] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:12:51.266Z I - [conn205722] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:12:51.275Z I CONTROL [conn205722] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:13:11.248Z I - [conn205628] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:13:11.257Z I CONTROL [conn205628] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:13:31.257Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:13:31.267Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:13:51.257Z I - [conn205872] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:13:51.267Z I CONTROL [conn205872] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:14:11.253Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:14:11.263Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:14:31.249Z I - [conn205471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:14:31.258Z I CONTROL [conn205471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:14:51.405Z I - [conn215858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:14:51.415Z I CONTROL [conn215858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:15:11.247Z I - [conn215922] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:15:11.257Z I CONTROL [conn215922] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:15:31.261Z I - [conn183087] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:15:31.270Z I CONTROL [conn183087] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:15:51.259Z I - [conn183115] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:15:51.268Z I CONTROL [conn183115] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:16:11.259Z I - [conn212599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:16:11.269Z I CONTROL [conn212599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:16:31.260Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:16:31.269Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:16:51.256Z I - [conn205638] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:16:51.266Z I CONTROL [conn205638] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:17:11.252Z I - [conn205638] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:17:11.261Z I CONTROL [conn205638] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:17:31.258Z I - [conn205633] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:17:31.268Z I CONTROL [conn205633] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:17:51.257Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:17:51.266Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:18:11.267Z I - [conn205633] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:18:11.277Z I CONTROL [conn205633] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:18:31.256Z I - [conn215874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:18:31.266Z I CONTROL [conn215874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:18:51.251Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:18:51.261Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:19:11.256Z I - [conn205853] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:19:11.266Z I CONTROL [conn205853] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:19:31.254Z I - [conn216078] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:19:31.264Z I CONTROL [conn216078] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:19:51.263Z I - [conn215971] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:19:51.273Z I CONTROL [conn215971] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:20:11.247Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:20:11.257Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:20:31.256Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:20:31.265Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:20:51.253Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:20:51.263Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:21:11.245Z I - [conn183115] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:21:11.255Z I CONTROL [conn183115] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:21:31.254Z I - [conn205666] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:21:31.264Z I CONTROL [conn205666] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:21:51.251Z I - [conn205872] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:21:51.261Z I CONTROL [conn205872] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:22:11.249Z I - [conn215976] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:22:11.259Z I CONTROL [conn215976] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:22:31.240Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:22:31.250Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:22:51.256Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:22:51.266Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:23:11.243Z I - [conn216091] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:23:11.252Z I CONTROL [conn216091] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:23:31.247Z I - [conn216135] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:23:31.256Z I CONTROL [conn216135] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:23:51.250Z I - [conn215911] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:23:51.259Z I CONTROL [conn215911] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:24:11.244Z I - [conn205878] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:24:11.253Z I CONTROL [conn205878] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:24:31.252Z I - [conn205584] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:24:31.261Z I CONTROL [conn205584] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:24:51.253Z I - [conn205722] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:24:51.263Z I CONTROL [conn205722] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:25:11.241Z I - [conn215144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:25:11.251Z I CONTROL [conn215144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:25:31.257Z I - [conn206298] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:25:31.266Z I CONTROL [conn206298] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:25:51.252Z I - [conn205941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:25:51.261Z I CONTROL [conn205941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:26:11.258Z I - [conn216156] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:26:11.267Z I CONTROL [conn216156] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:26:31.249Z I - [conn216117] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:26:31.258Z I CONTROL [conn216117] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:26:51.248Z I - [conn205796] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:26:51.258Z I CONTROL [conn205796] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:27:11.245Z I - [conn206330] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:27:11.254Z I CONTROL [conn206330] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:27:31.252Z I - [conn205904] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:27:31.262Z I CONTROL [conn205904] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:27:51.257Z I - [conn205606] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:27:51.267Z I CONTROL [conn205606] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:28:11.253Z I - [conn206768] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:28:11.262Z I CONTROL [conn206768] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:28:31.246Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:28:31.256Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:28:51.242Z I - [conn215982] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:28:51.252Z I CONTROL [conn215982] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:29:11.253Z I - [conn215930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:29:11.263Z I CONTROL [conn215930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:29:31.251Z I - [conn205528] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:29:31.261Z I CONTROL [conn205528] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:29:51.246Z I - [conn205920] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:29:51.255Z I CONTROL [conn205920] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:30:11.259Z I - [conn205920] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:30:11.269Z I CONTROL [conn205920] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:30:31.241Z I - [conn215850] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:30:31.251Z I CONTROL [conn215850] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:30:51.248Z I - [conn205421] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:30:51.258Z I CONTROL [conn205421] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:31:11.251Z I - [conn207417] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:31:11.260Z I CONTROL [conn207417] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:31:31.249Z I - [conn207402] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:31:31.259Z I CONTROL [conn207402] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:31:51.233Z I - [conn215924] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:31:51.242Z I CONTROL [conn215924] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:32:11.239Z I - [conn212226] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:32:11.249Z I CONTROL [conn212226] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:32:31.246Z I - [conn183087] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:32:31.255Z I CONTROL [conn183087] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:32:51.246Z I - [conn205562] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:32:51.256Z I CONTROL [conn205562] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:33:11.259Z I - [conn216066] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:33:11.268Z I CONTROL [conn216066] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:33:31.247Z I - [conn206298] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:33:31.257Z I CONTROL [conn206298] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:33:51.249Z I - [conn205743] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:33:51.258Z I CONTROL [conn205743] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:34:11.234Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:34:11.244Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:34:31.242Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:34:31.252Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:34:51.242Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:34:51.252Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:35:11.231Z I - [conn216127] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:35:11.241Z I CONTROL [conn216127] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:35:31.235Z I - [conn215858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:35:31.244Z I CONTROL [conn215858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:35:51.241Z I - [conn216061] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:35:51.251Z I CONTROL [conn216061] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:36:11.239Z I - [conn216148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:36:11.249Z I CONTROL [conn216148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:36:31.232Z I - [conn215947] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:36:31.242Z I CONTROL [conn215947] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:36:51.231Z I - [conn216145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:36:51.241Z I CONTROL [conn216145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:37:11.242Z I - [conn205664] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:37:11.252Z I CONTROL [conn205664] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:37:31.246Z I - [conn206257] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:37:31.257Z I CONTROL [conn206257] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:37:51.242Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:37:51.251Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:38:11.245Z I - [conn205931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:38:11.254Z I CONTROL [conn205931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:38:31.245Z I - [conn205948] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:38:31.254Z I CONTROL [conn205948] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:38:51.245Z I - [conn206800] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:38:51.254Z I CONTROL [conn206800] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:39:11.233Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:39:11.243Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:39:31.243Z I - [conn205719] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:39:31.254Z I CONTROL [conn205719] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:39:51.240Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:39:51.250Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:40:11.252Z I - [conn205697] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:40:11.262Z I CONTROL [conn205697] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:40:31.250Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:40:31.259Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:40:51.236Z I - [conn215977] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:40:51.246Z I CONTROL [conn215977] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:41:11.245Z I - [conn205743] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:41:11.254Z I CONTROL [conn205743] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:41:31.256Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:41:31.265Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:41:43.673Z I - [conn206255] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:41:43.683Z I CONTROL [conn206255] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:41:51.230Z I - [conn216132] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:41:51.240Z I CONTROL [conn216132] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:42:11.237Z I - [conn215917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:42:11.246Z I CONTROL [conn215917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:42:31.230Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:42:31.239Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:42:51.245Z I - [conn205942] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:42:51.255Z I CONTROL [conn205942] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:43:11.240Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:43:11.250Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:43:31.250Z I - [conn205737] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:43:31.259Z I CONTROL [conn205737] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:43:51.241Z I - [conn205484] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:43:51.251Z I CONTROL [conn205484] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:44:11.242Z I - [conn205815] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:44:11.252Z I CONTROL [conn205815] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:44:31.231Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:44:31.241Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:44:51.242Z I - [conn215930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:44:51.252Z I CONTROL [conn215930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:45:11.243Z I - [conn205664] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:45:11.252Z I CONTROL [conn205664] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:45:31.229Z I - [conn205737] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:45:31.239Z I CONTROL [conn205737] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:45:51.239Z I - [conn205931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:45:51.248Z I CONTROL [conn205931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:46:11.242Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:46:11.252Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:46:31.238Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:46:31.247Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:46:51.235Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:46:51.244Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:47:11.253Z I - [conn205853] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:47:11.264Z I CONTROL [conn205853] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:47:31.227Z I - [conn205737] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:47:31.236Z I CONTROL [conn205737] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:47:51.234Z I - [conn215978] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:47:51.243Z I CONTROL [conn215978] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:48:11.238Z I - [conn205844] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:48:11.249Z I CONTROL [conn205844] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:48:31.233Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:48:31.242Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:48:51.233Z I - [conn216148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:48:51.243Z I CONTROL [conn216148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:49:11.235Z I - [conn206774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:49:11.245Z I CONTROL [conn206774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:49:31.235Z I - [conn215943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:49:31.244Z I CONTROL [conn215943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:49:51.229Z I - [conn216144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:49:51.239Z I CONTROL [conn216144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:50:11.238Z I - [conn206800] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:50:11.247Z I CONTROL [conn206800] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:50:31.238Z I - [conn205548] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:50:31.247Z I CONTROL [conn205548] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:50:51.235Z I - [conn216060] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:50:51.245Z I CONTROL [conn216060] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:51:11.237Z I - [conn215908] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:51:11.247Z I CONTROL [conn215908] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:51:31.233Z I - [conn205572] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:51:31.243Z I CONTROL [conn205572] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:51:51.240Z I - [conn205920] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:51:51.249Z I CONTROL [conn205920] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:52:11.235Z I - [conn212229] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:52:11.245Z I CONTROL [conn212229] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:52:31.237Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:52:31.247Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:52:51.238Z I - [conn205473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:52:51.248Z I CONTROL [conn205473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:53:11.226Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:53:11.236Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:53:31.230Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:53:31.240Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:53:51.240Z I - [conn206768] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:53:51.250Z I CONTROL [conn206768] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:54:11.223Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:54:11.233Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:54:31.234Z I - [conn183088] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:54:31.243Z I CONTROL [conn183088] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:54:51.239Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:54:51.248Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:55:11.234Z I - [conn177419] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:55:11.243Z I CONTROL [conn177419] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:55:31.234Z I - [conn216060] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:55:31.244Z I CONTROL [conn216060] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:55:51.237Z I - [conn212601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:55:51.247Z I CONTROL [conn212601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:56:11.222Z I - [conn216145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:56:11.231Z I CONTROL [conn216145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:56:31.244Z I - [conn215930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:56:31.253Z I CONTROL [conn215930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:56:51.225Z I - [conn216115] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:56:51.235Z I CONTROL [conn216115] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:57:11.228Z I - [conn206311] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:57:11.237Z I CONTROL [conn206311] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:57:31.242Z I - [conn205878] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:57:31.252Z I CONTROL [conn205878] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:57:51.227Z I - [conn205842] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:57:51.236Z I CONTROL [conn205842] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:58:11.238Z I - [conn205740] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:58:11.248Z I CONTROL [conn205740] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:58:31.238Z I - [conn216136] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:58:31.248Z I CONTROL [conn216136] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:58:51.234Z I - [conn207685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:58:51.244Z I CONTROL [conn207685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:59:11.220Z I - [conn216132] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:59:11.230Z I CONTROL [conn216132] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:59:31.230Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:59:31.240Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:59:51.225Z I - [conn170473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T09:59:51.235Z I CONTROL [conn170473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T09:59:51.434Z I INDEX [conn215461] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T10:00:11.233Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:00:11.243Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:00:31.226Z I - [conn215950] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:00:31.236Z I CONTROL [conn215950] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:00:51.241Z I - [conn205831] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:00:51.251Z I CONTROL [conn205831] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:01:11.219Z I - [conn205810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:01:11.229Z I CONTROL [conn205810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:01:31.224Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:01:31.234Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:01:51.244Z I - [conn205595] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:01:51.253Z I CONTROL [conn205595] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:02:11.223Z I - [conn205748] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:02:11.232Z I CONTROL [conn205748] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:02:31.218Z I - [conn215976] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:02:31.228Z I CONTROL [conn215976] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:02:51.230Z I - [conn205659] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:02:51.239Z I CONTROL [conn205659] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:03:11.229Z I - [conn206768] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:03:11.238Z I CONTROL [conn206768] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:03:31.246Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:03:31.255Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:03:51.231Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:03:51.240Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:04:11.238Z I - [conn215876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:04:11.249Z I CONTROL [conn215876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:04:31.239Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:04:31.249Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:04:51.221Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:04:51.231Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:05:11.228Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:05:11.238Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:05:31.223Z I - [conn206319] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:05:31.232Z I CONTROL [conn206319] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:05:51.223Z I - [conn216110] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:05:51.233Z I CONTROL [conn216110] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:06:11.229Z I - [conn205499] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:06:11.238Z I CONTROL [conn205499] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:06:31.231Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:06:31.241Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:06:51.231Z I - [conn215968] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:06:51.241Z I CONTROL [conn215968] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:07:11.232Z I - [conn215968] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:07:11.242Z I CONTROL [conn215968] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:07:31.228Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:07:31.238Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:07:51.232Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:07:51.241Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:08:11.225Z I - [conn215913] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:08:11.235Z I CONTROL [conn215913] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:08:31.231Z I - [conn207408] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:08:31.240Z I CONTROL [conn207408] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:08:51.217Z I - [conn207401] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:08:51.227Z I CONTROL [conn207401] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:09:11.232Z I - [conn205774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:09:11.242Z I CONTROL [conn205774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:09:31.224Z I - [conn205737] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:09:31.233Z I CONTROL [conn205737] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:09:51.240Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:09:51.250Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:10:11.228Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:10:11.238Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:10:31.221Z I - [conn215755] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:10:31.230Z I CONTROL [conn215755] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:10:49.647Z I - [conn205915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:10:49.657Z I CONTROL [conn205915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:10:51.225Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:10:51.234Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:11:11.228Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:11:11.238Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:11:31.225Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:11:31.234Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:11:51.234Z I - [conn205562] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:11:51.244Z I CONTROL [conn205562] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:12:11.215Z I - [conn205471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:12:11.224Z I CONTROL [conn205471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:12:31.228Z I - [conn215147] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:12:31.238Z I CONTROL [conn215147] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:12:51.215Z I - [conn205736] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:12:51.224Z I CONTROL [conn205736] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:13:11.224Z I - [conn205801] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:13:11.234Z I CONTROL [conn205801] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:13:31.243Z I - [conn206329] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:13:31.253Z I CONTROL [conn206329] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:13:51.224Z I - [conn205882] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:13:51.234Z I CONTROL [conn205882] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:14:11.216Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:14:11.225Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:14:31.213Z I - [conn215943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:14:31.223Z I CONTROL [conn215943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:14:51.233Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:14:51.243Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:15:11.224Z I - [conn216113] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:15:11.234Z I CONTROL [conn216113] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:15:31.222Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:15:31.232Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:15:51.215Z I - [conn205811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:15:51.224Z I CONTROL [conn205811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:16:11.224Z I - [conn205517] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:16:11.234Z I CONTROL [conn205517] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:16:31.224Z I - [conn183087] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:16:31.233Z I CONTROL [conn183087] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:16:51.210Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:16:51.219Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:17:11.212Z I - [conn206794] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:17:11.222Z I CONTROL [conn206794] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:17:31.223Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:17:31.233Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:17:51.221Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:17:51.230Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:18:11.210Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:18:11.220Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:18:31.222Z I - [conn206774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:18:31.231Z I CONTROL [conn206774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:18:51.212Z I - [conn206319] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:18:51.222Z I CONTROL [conn206319] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:19:11.211Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:19:11.220Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:19:31.222Z I - [conn206794] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:19:31.232Z I CONTROL [conn206794] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:19:51.219Z I - [conn216148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:19:51.229Z I CONTROL [conn216148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:20:11.223Z I - [conn216148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:20:11.233Z I CONTROL [conn216148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:20:31.221Z I - [conn206330] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:20:31.230Z I CONTROL [conn206330] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:20:51.221Z I - [conn205712] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:20:51.230Z I CONTROL [conn205712] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:21:11.219Z I - [conn206329] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:21:11.228Z I CONTROL [conn206329] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:21:24.716Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:21:24.725Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:21:31.215Z I - [conn215927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:21:31.225Z I CONTROL [conn215927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:21:51.216Z I - [conn205570] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:21:51.226Z I CONTROL [conn205570] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:22:11.217Z I - [conn215973] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:22:11.227Z I CONTROL [conn215973] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:22:31.206Z I - [conn215975] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:22:31.216Z I CONTROL [conn215975] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:22:51.219Z I - [conn170473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:22:51.228Z I CONTROL [conn170473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:23:11.219Z I - [conn205568] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:23:11.229Z I CONTROL [conn205568] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:23:31.208Z I - [conn205831] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:23:31.217Z I CONTROL [conn205831] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:23:51.224Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:23:51.234Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:24:11.209Z I - [conn215956] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:24:11.218Z I CONTROL [conn215956] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:24:31.215Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:24:31.226Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:24:51.222Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:24:51.232Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:25:11.217Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:25:11.227Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:25:31.218Z I - [conn205671] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:25:31.228Z I CONTROL [conn205671] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:25:51.206Z I - [conn205871] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:25:51.216Z I CONTROL [conn205871] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:26:11.222Z I - [conn170898] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:26:11.231Z I CONTROL [conn170898] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:26:31.218Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:26:31.228Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:26:51.212Z I - [conn205568] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:26:51.222Z I CONTROL [conn205568] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:27:11.228Z I - [conn205848] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:27:11.237Z I CONTROL [conn205848] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:27:31.220Z I - [conn205937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:27:31.229Z I CONTROL [conn205937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:27:51.216Z I - [conn205901] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:27:51.225Z I CONTROL [conn205901] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:28:11.220Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:28:11.229Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:28:31.213Z I - [conn215880] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:28:31.223Z I CONTROL [conn215880] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:28:51.215Z I - [conn205424] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:28:51.225Z I CONTROL [conn205424] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:29:11.219Z I - [conn206774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:29:11.228Z I CONTROL [conn206774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:29:31.217Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:29:31.226Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:29:51.345Z I - [conn205918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:29:51.354Z I CONTROL [conn205918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:30:11.202Z I - [conn205740] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:30:11.211Z I CONTROL [conn205740] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:30:31.210Z I - [conn216107] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:30:31.219Z I CONTROL [conn216107] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:30:51.209Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:30:51.219Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:31:01.836Z I - [conn206255] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:31:01.846Z I CONTROL [conn206255] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:31:11.212Z I - [conn216138] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:31:11.222Z I CONTROL [conn216138] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:31:31.233Z I - [conn177379] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:31:31.243Z I CONTROL [conn177379] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:31:51.210Z I - [conn216144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:31:51.220Z I CONTROL [conn216144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:32:11.211Z I - [conn205915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:32:11.220Z I CONTROL [conn205915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:32:31.202Z I - [conn215927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:32:31.212Z I CONTROL [conn215927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:32:51.213Z I - [conn205810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:32:51.222Z I CONTROL [conn205810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:33:11.209Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:33:11.219Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:33:31.201Z I - [conn205697] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:33:31.211Z I CONTROL [conn205697] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:33:51.206Z I - [conn205518] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:33:51.216Z I CONTROL [conn205518] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:34:11.213Z I - [conn205822] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:34:11.223Z I CONTROL [conn205822] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:34:31.214Z I - [conn206316] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:34:31.224Z I CONTROL [conn206316] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:34:51.212Z I - [conn206328] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:34:51.222Z I CONTROL [conn206328] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:35:11.220Z I - [conn205943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:35:11.230Z I CONTROL [conn205943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:35:31.200Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:35:31.209Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:35:51.212Z I - [conn216144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:35:51.222Z I CONTROL [conn216144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:36:11.205Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:36:11.214Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:36:31.215Z I - [conn205945] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:36:31.225Z I CONTROL [conn205945] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:36:51.202Z I - [conn205478] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:36:51.212Z I CONTROL [conn205478] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:37:11.208Z I - [conn216095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:37:11.217Z I CONTROL [conn216095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:37:31.205Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:37:31.214Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:37:51.210Z I - [conn205626] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:37:51.219Z I CONTROL [conn205626] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:38:11.208Z I - [conn216145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:38:11.218Z I CONTROL [conn216145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:38:31.206Z I - [conn205874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:38:31.215Z I CONTROL [conn205874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:38:51.213Z I - [conn216131] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:38:51.222Z I CONTROL [conn216131] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:39:11.220Z I - [conn205837] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:39:11.230Z I CONTROL [conn205837] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:39:31.208Z I - [conn216124] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:39:31.218Z I CONTROL [conn216124] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:39:51.198Z I - [conn205587] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:39:51.208Z I CONTROL [conn205587] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:40:11.210Z I - [conn177419] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:40:11.220Z I CONTROL [conn177419] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:40:31.200Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:40:31.209Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:40:51.199Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:40:51.209Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:41:11.210Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:41:11.220Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:41:31.209Z I - [conn205728] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:41:31.219Z I CONTROL [conn205728] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:41:51.200Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:41:51.210Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:42:11.218Z I - [conn216072] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:42:11.227Z I CONTROL [conn216072] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:42:31.200Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:42:31.209Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:42:51.209Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:42:51.219Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:43:11.208Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:43:11.217Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:43:31.209Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:43:31.218Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:43:51.207Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:43:51.216Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:44:11.209Z I - [conn205948] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:44:11.219Z I CONTROL [conn205948] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:44:31.198Z I - [conn205457] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:44:31.208Z I CONTROL [conn205457] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:44:51.203Z I - [conn207402] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:44:51.213Z I CONTROL [conn207402] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:45:11.209Z I - [conn206768] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:45:11.218Z I CONTROL [conn206768] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:45:31.197Z I - [conn205837] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:45:31.207Z I CONTROL [conn205837] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:45:51.211Z I - [conn207410] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:45:51.221Z I CONTROL [conn207410] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:46:11.211Z I - [conn212598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:46:11.221Z I CONTROL [conn212598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:46:31.206Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:46:31.216Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:46:51.202Z I - [conn215979] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:46:51.212Z I CONTROL [conn215979] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:47:11.204Z I - [conn183343] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:47:11.213Z I CONTROL [conn183343] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:47:31.205Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:47:31.214Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:47:51.202Z I - [conn215979] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:47:51.211Z I CONTROL [conn215979] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:48:11.206Z I - [conn205628] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:48:11.215Z I CONTROL [conn205628] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:48:31.211Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:48:31.220Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:48:51.195Z I - [conn205904] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:48:51.205Z I CONTROL [conn205904] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:49:11.204Z I - [conn215979] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:49:11.214Z I CONTROL [conn215979] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:49:31.220Z I - [conn205658] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:49:31.229Z I CONTROL [conn205658] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:49:51.204Z I - [conn216112] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:49:51.213Z I CONTROL [conn216112] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:50:11.201Z I - [conn205853] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:50:11.211Z I CONTROL [conn205853] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:50:31.210Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:50:31.219Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:50:51.195Z I - [conn205863] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:50:51.205Z I CONTROL [conn205863] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:51:11.208Z I - [conn205917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:51:11.218Z I CONTROL [conn205917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:51:31.204Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:51:31.214Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:51:51.198Z I - [conn215962] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:51:51.208Z I CONTROL [conn215962] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:52:11.197Z I - [conn216145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:52:11.206Z I CONTROL [conn216145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:52:31.212Z I - [conn205941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:52:31.221Z I CONTROL [conn205941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:52:51.206Z I - [conn205625] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:52:51.216Z I CONTROL [conn205625] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:53:11.205Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:53:11.215Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:53:31.200Z I - [conn205943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:53:31.209Z I CONTROL [conn205943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:53:51.226Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:53:51.235Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:54:11.206Z I - [conn216108] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:54:11.216Z I CONTROL [conn216108] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:54:31.210Z I - [conn205749] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:54:31.220Z I CONTROL [conn205749] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:54:51.196Z I - [conn215974] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:54:51.205Z I CONTROL [conn215974] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:55:11.207Z I - [conn206319] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:55:11.216Z I CONTROL [conn206319] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:55:31.198Z I - [conn206319] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:55:31.208Z I CONTROL [conn206319] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:55:51.195Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:55:51.205Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:56:11.200Z I - [conn215983] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:56:11.210Z I CONTROL [conn215983] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:56:31.194Z I - [conn206313] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:56:31.204Z I CONTROL [conn206313] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:56:51.201Z I - [conn215944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:56:51.211Z I CONTROL [conn215944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:57:11.202Z I - [conn215944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:57:11.212Z I CONTROL [conn215944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:57:31.204Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:57:31.213Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:57:51.308Z I - [conn206775] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:57:51.317Z I CONTROL [conn206775] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:58:11.205Z I - [conn206314] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:58:11.215Z I CONTROL [conn206314] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:58:31.224Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:58:31.233Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:58:51.191Z I - [conn205697] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:58:51.200Z I CONTROL [conn205697] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:59:11.202Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:59:11.212Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:59:31.202Z I - [conn215880] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:59:31.212Z I CONTROL [conn215880] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T10:59:48.913Z I INDEX [conn215568] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T10:59:51.197Z I - [conn216097] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T10:59:51.206Z I CONTROL [conn216097] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:00:11.203Z I - [conn205829] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:00:11.213Z I CONTROL [conn205829] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:00:31.208Z I - [conn205822] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:00:31.218Z I CONTROL [conn205822] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:00:51.195Z I - [conn205671] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:00:51.206Z I CONTROL [conn205671] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:01:11.205Z I - [conn205482] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:01:11.215Z I CONTROL [conn205482] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:01:31.189Z I - [conn215855] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:01:31.199Z I CONTROL [conn215855] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:01:51.199Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:01:51.209Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:02:11.201Z I - [conn215876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:02:11.210Z I CONTROL [conn215876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:02:31.205Z I - [conn215876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:02:31.215Z I CONTROL [conn215876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:02:51.199Z I - [conn215983] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:02:51.209Z I CONTROL [conn215983] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:03:11.193Z I - [conn216107] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:03:11.203Z I CONTROL [conn216107] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:03:31.206Z I - [conn215133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:03:31.216Z I CONTROL [conn215133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:03:51.200Z I - [conn205587] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:03:51.210Z I CONTROL [conn205587] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:04:11.208Z I - [conn216141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:04:11.217Z I CONTROL [conn216141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:04:31.192Z I - [conn183343] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:04:31.202Z I CONTROL [conn183343] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:04:51.199Z I - [conn215859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:04:51.209Z I CONTROL [conn215859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:05:11.188Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:05:11.197Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:05:31.202Z I - [conn206256] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:05:31.211Z I CONTROL [conn206256] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:05:51.188Z I - [conn215849] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:05:51.197Z I CONTROL [conn215849] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:06:11.203Z I - [conn205424] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:06:11.212Z I CONTROL [conn205424] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:06:31.202Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:06:31.211Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:06:51.188Z I - [conn205617] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:06:51.198Z I CONTROL [conn205617] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:07:11.189Z I - [conn212600] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:07:11.198Z I CONTROL [conn212600] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:07:31.187Z I - [conn216107] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:07:31.197Z I CONTROL [conn216107] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:07:51.195Z I - [conn216099] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:07:51.204Z I CONTROL [conn216099] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:08:11.192Z I - [conn216099] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:08:11.202Z I CONTROL [conn216099] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:08:31.355Z I - [conn170487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:08:31.365Z I CONTROL [conn170487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:08:51.197Z I - [conn216153] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:08:51.207Z I CONTROL [conn216153] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:09:11.198Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:09:11.208Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:09:31.197Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:09:31.206Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:09:51.198Z I - [conn205517] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:09:51.208Z I CONTROL [conn205517] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:10:11.204Z I - [conn206320] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:10:11.213Z I CONTROL [conn206320] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:10:31.201Z I - [conn183099] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:10:31.211Z I CONTROL [conn183099] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:10:51.196Z I - [conn205548] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:10:51.206Z I CONTROL [conn205548] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:11:11.184Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:11:11.193Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:11:31.197Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:11:31.207Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:11:51.198Z I - [conn206797] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:11:51.208Z I CONTROL [conn206797] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:12:11.189Z I - [conn207685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:12:11.199Z I CONTROL [conn207685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:12:31.185Z I - [conn205628] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:12:31.195Z I CONTROL [conn205628] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:12:51.183Z I - [conn205628] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:12:51.193Z I CONTROL [conn205628] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:13:11.198Z I - [conn216118] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:13:11.208Z I CONTROL [conn216118] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:13:31.198Z I - [conn215960] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:13:31.208Z I CONTROL [conn215960] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:13:51.187Z I - [conn170487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:13:51.197Z I CONTROL [conn170487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:14:11.184Z I - [conn205871] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:14:11.194Z I CONTROL [conn205871] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:14:31.200Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:14:31.209Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:14:51.184Z I - [conn205749] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:14:51.194Z I CONTROL [conn205749] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:15:11.194Z I - [conn183087] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:15:11.204Z I CONTROL [conn183087] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:15:31.185Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:15:31.194Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:15:51.187Z I - [conn179981] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:15:51.197Z I CONTROL [conn179981] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:16:11.195Z I - [conn205831] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:16:11.205Z I CONTROL [conn205831] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:16:31.187Z I - [conn215140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:16:31.197Z I CONTROL [conn215140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:16:51.193Z I - [conn205848] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:16:51.203Z I CONTROL [conn205848] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:16:56.336Z I - [conn206794] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:16:56.345Z I CONTROL [conn206794] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:17:11.184Z I - [conn205577] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:17:11.194Z I CONTROL [conn205577] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:17:31.205Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:17:31.215Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:17:51.301Z I - [conn215970] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:17:51.311Z I CONTROL [conn215970] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:18:11.194Z I - [conn207408] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:18:11.203Z I CONTROL [conn207408] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:18:31.189Z I - [conn205920] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:18:31.199Z I CONTROL [conn205920] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:18:51.190Z I - [conn215943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:18:51.200Z I CONTROL [conn215943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:19:11.192Z I - [conn215943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:19:11.201Z I CONTROL [conn215943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:19:31.191Z I - [conn177404] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:19:31.201Z I CONTROL [conn177404] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:19:51.193Z I - [conn205901] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:19:51.203Z I CONTROL [conn205901] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:20:11.189Z I - [conn205871] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:20:11.199Z I CONTROL [conn205871] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:20:31.187Z I - [conn183336] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:20:31.197Z I CONTROL [conn183336] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:20:51.197Z I - [conn216105] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:20:51.207Z I CONTROL [conn216105] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:21:11.190Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:21:11.199Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:21:31.190Z I - [conn215874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:21:31.200Z I CONTROL [conn215874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:21:51.200Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:21:51.210Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:22:11.202Z I - [conn170898] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:22:11.213Z I CONTROL [conn170898] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:22:31.196Z I - [conn206774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:22:31.205Z I CONTROL [conn206774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:22:51.188Z I - [conn205697] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:22:51.198Z I CONTROL [conn205697] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:23:11.191Z I - [conn205595] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:23:11.201Z I CONTROL [conn205595] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:23:31.188Z I - [conn216118] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:23:31.198Z I CONTROL [conn216118] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:23:51.191Z I - [conn183099] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:23:51.200Z I CONTROL [conn183099] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:24:11.189Z I - [conn215133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:24:11.199Z I CONTROL [conn215133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:24:31.189Z I - [conn205634] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:24:31.199Z I CONTROL [conn205634] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:24:51.187Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:24:51.196Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:25:11.190Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:25:11.200Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:25:31.193Z I - [conn205904] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:25:31.202Z I CONTROL [conn205904] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:25:51.185Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:25:51.195Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:26:11.190Z I - [conn205712] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:26:11.200Z I CONTROL [conn205712] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:26:31.178Z I - [conn205712] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:26:31.187Z I CONTROL [conn205712] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:26:51.188Z I - [conn216081] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:26:51.199Z I CONTROL [conn216081] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:27:11.196Z I - [conn216061] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:27:11.206Z I CONTROL [conn216061] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:27:31.179Z I - [conn215959] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:27:31.189Z I CONTROL [conn215959] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:27:51.185Z I - [conn205765] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:27:51.195Z I CONTROL [conn205765] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:28:11.195Z I - [conn206298] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:28:11.204Z I CONTROL [conn206298] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:28:31.188Z I - [conn215974] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:28:31.198Z I CONTROL [conn215974] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:28:51.190Z I - [conn205554] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:28:51.200Z I CONTROL [conn205554] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:29:11.176Z I - [conn205900] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:29:11.185Z I CONTROL [conn205900] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:29:31.176Z I - [conn205852] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:29:31.186Z I CONTROL [conn205852] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:29:51.189Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:29:51.199Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:30:11.184Z I - [conn215926] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:30:11.193Z I CONTROL [conn215926] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:30:31.189Z I - [conn205639] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:30:31.199Z I CONTROL [conn205639] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:30:51.185Z I - [conn206330] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:30:51.195Z I CONTROL [conn206330] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:31:11.174Z I - [conn207416] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:31:11.184Z I CONTROL [conn207416] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:31:31.179Z I - [conn207416] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:31:31.188Z I CONTROL [conn207416] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:31:51.177Z I - [conn207416] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:31:51.187Z I CONTROL [conn207416] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:32:11.171Z I - [conn216145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:32:11.180Z I CONTROL [conn216145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:32:31.183Z I - [conn215859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:32:31.193Z I CONTROL [conn215859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:32:51.169Z I - [conn216097] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:32:51.179Z I CONTROL [conn216097] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:33:11.186Z I - [conn215921] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:33:11.196Z I CONTROL [conn215921] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:33:31.181Z I - [conn205842] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:33:31.191Z I CONTROL [conn205842] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:33:51.185Z I - [conn205598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:33:51.194Z I CONTROL [conn205598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:34:11.184Z I - [conn205497] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:34:11.194Z I CONTROL [conn205497] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:34:31.193Z I - [conn216115] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:34:31.203Z I CONTROL [conn216115] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:34:51.186Z I - [conn177468] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:34:51.196Z I CONTROL [conn177468] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:35:11.197Z I - [conn207410] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:35:11.207Z I CONTROL [conn207410] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:35:31.188Z I - [conn215874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:35:31.198Z I CONTROL [conn215874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:35:41.124Z I - [conn205743] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:35:41.134Z I CONTROL [conn205743] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:35:51.186Z I - [conn207401] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:35:51.196Z I CONTROL [conn207401] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:36:11.175Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:36:11.184Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:36:31.178Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:36:31.187Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:36:49.102Z I - [conn206317] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:36:49.112Z I CONTROL [conn206317] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:36:51.174Z I - [conn205628] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:36:51.183Z I CONTROL [conn205628] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:37:11.175Z I - [conn205880] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:37:11.184Z I CONTROL [conn205880] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:37:24.286Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:37:24.295Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:37:31.179Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:37:31.188Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:37:51.187Z I - [conn205736] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:37:51.197Z I CONTROL [conn205736] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:38:11.182Z I - [conn215932] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:38:11.191Z I CONTROL [conn215932] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:38:31.191Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:38:31.201Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:38:51.169Z I - [conn205868] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:38:51.179Z I CONTROL [conn205868] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:39:11.178Z I - [conn205484] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:39:11.188Z I CONTROL [conn205484] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:39:31.171Z I - [conn205791] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:39:31.180Z I CONTROL [conn205791] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:39:51.169Z I - [conn205562] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:39:51.179Z I CONTROL [conn205562] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:40:11.180Z I - [conn205743] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:40:11.190Z I CONTROL [conn205743] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:40:31.180Z I - [conn183103] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:40:31.189Z I CONTROL [conn183103] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:40:51.173Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:40:51.183Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:41:11.177Z I - [conn183087] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:41:11.187Z I CONTROL [conn183087] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:41:31.172Z I - [conn215971] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:41:31.182Z I CONTROL [conn215971] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:41:32.562Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:41:32.571Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:41:51.167Z I - [conn205918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:41:51.176Z I CONTROL [conn205918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:42:11.177Z I - [conn205840] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:42:11.186Z I CONTROL [conn205840] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:42:31.180Z I - [conn206315] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:42:31.190Z I CONTROL [conn206315] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:42:51.179Z I - [conn207401] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:42:51.189Z I CONTROL [conn207401] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:43:11.168Z I - [conn215927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:43:11.178Z I CONTROL [conn215927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:43:31.179Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:43:31.188Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:43:51.175Z I - [conn216104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:43:51.185Z I CONTROL [conn216104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:44:11.170Z I - [conn205920] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:44:11.180Z I CONTROL [conn205920] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:44:31.166Z I - [conn205722] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:44:31.175Z I CONTROL [conn205722] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:44:51.177Z I - [conn205935] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:44:51.188Z I CONTROL [conn205935] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:45:11.165Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:45:11.175Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:45:27.331Z I - [conn216123] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:45:27.341Z I CONTROL [conn216123] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:45:31.172Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:45:31.182Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:45:51.190Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:45:51.200Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:46:11.168Z I - [conn206256] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:46:11.177Z I CONTROL [conn206256] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:46:31.177Z I - [conn177419] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:46:31.186Z I CONTROL [conn177419] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:46:51.173Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:46:51.183Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:47:11.175Z I - [conn177419] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:47:11.184Z I CONTROL [conn177419] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:47:31.171Z I - [conn215976] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:47:31.181Z I CONTROL [conn215976] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:47:51.174Z I - [conn205853] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:47:51.184Z I CONTROL [conn205853] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:48:11.177Z I - [conn205945] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:48:11.187Z I CONTROL [conn205945] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:48:31.179Z I - [conn205675] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:48:31.189Z I CONTROL [conn205675] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:48:32.846Z I - [conn206800] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:48:32.856Z I CONTROL [conn206800] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:48:51.168Z I - [conn205554] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:48:51.178Z I CONTROL [conn205554] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:49:11.175Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:49:11.185Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:49:31.174Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:49:31.184Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:49:51.175Z I - [conn205871] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:49:51.185Z I CONTROL [conn205871] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:50:11.173Z I - [conn205945] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:50:11.183Z I CONTROL [conn205945] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:50:31.180Z I - [conn205604] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:50:31.189Z I CONTROL [conn205604] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:50:51.427Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:50:51.438Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:51:10.256Z I - [conn212598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:51:10.266Z I CONTROL [conn212598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:51:11.171Z I - [conn205787] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:51:11.181Z I CONTROL [conn205787] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:51:31.163Z I - [conn206811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:51:31.172Z I CONTROL [conn206811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:51:51.171Z I - [conn216052] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:51:51.181Z I CONTROL [conn216052] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:52:11.173Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:52:11.183Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:52:31.159Z I - [conn205869] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:52:31.169Z I CONTROL [conn205869] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:52:51.172Z I - [conn206317] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:52:51.181Z I CONTROL [conn206317] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:53:11.174Z I - [conn206314] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:53:11.183Z I CONTROL [conn206314] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:53:31.163Z I - [conn216122] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:53:31.173Z I CONTROL [conn216122] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:53:51.169Z I - [conn215960] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:53:51.178Z I CONTROL [conn215960] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:54:11.169Z I - [conn216062] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:54:11.179Z I CONTROL [conn216062] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:54:31.170Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:54:31.180Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:54:51.172Z I - [conn205815] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:54:51.181Z I CONTROL [conn205815] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:55:11.158Z I - [conn216095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:55:11.167Z I CONTROL [conn216095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:55:31.171Z I - [conn207410] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:55:31.181Z I CONTROL [conn207410] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:55:51.167Z I - [conn216080] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:55:51.177Z I CONTROL [conn216080] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:56:11.165Z I - [conn206314] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:56:11.175Z I CONTROL [conn206314] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:56:31.168Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:56:31.178Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:56:51.167Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:56:51.177Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:57:11.163Z I - [conn215974] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:57:11.173Z I CONTROL [conn215974] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:57:31.169Z I - [conn205848] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:57:31.179Z I CONTROL [conn205848] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:57:51.169Z I - [conn205528] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:57:51.178Z I CONTROL [conn205528] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:58:11.176Z I - [conn205707] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:58:11.186Z I CONTROL [conn205707] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:58:31.166Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:58:31.176Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:58:51.168Z I - [conn205628] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:58:51.178Z I CONTROL [conn205628] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:59:11.155Z I - [conn205587] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:59:11.164Z I CONTROL [conn205587] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:59:31.156Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:59:31.165Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T11:59:48.941Z I INDEX [conn215461] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T11:59:51.160Z I - [conn205500] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T11:59:51.171Z I CONTROL [conn205500] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:00:11.168Z I - [conn170471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:00:11.177Z I CONTROL [conn170471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:00:31.171Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:00:31.180Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:00:51.155Z I - [conn215973] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:00:51.165Z I CONTROL [conn215973] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:01:11.172Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:01:11.182Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:01:31.167Z I - [conn216107] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:01:31.177Z I CONTROL [conn216107] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:01:51.169Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:01:51.178Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:02:11.169Z I - [conn212225] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:02:11.178Z I CONTROL [conn212225] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:02:31.156Z I - [conn183336] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:02:31.166Z I CONTROL [conn183336] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:02:51.155Z I - [conn216110] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:02:51.164Z I CONTROL [conn216110] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:03:11.162Z I - [conn212600] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:03:11.171Z I CONTROL [conn212600] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:03:31.158Z I - [conn216127] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:03:31.168Z I CONTROL [conn216127] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:03:51.159Z I - [conn215858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:03:51.169Z I CONTROL [conn215858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:04:11.156Z I - [conn216125] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:04:11.166Z I CONTROL [conn216125] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:04:31.163Z I - [conn215982] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:04:31.172Z I CONTROL [conn215982] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:04:51.174Z I - [conn205615] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:04:51.184Z I CONTROL [conn205615] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:05:11.164Z I - [conn205482] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:05:11.173Z I CONTROL [conn205482] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:05:31.164Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:05:31.174Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:05:51.163Z I - [conn205760] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:05:51.173Z I CONTROL [conn205760] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:06:11.171Z I - [conn170471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:06:11.181Z I CONTROL [conn170471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:06:31.170Z I - [conn177467] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:06:31.179Z I CONTROL [conn177467] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:06:51.375Z I - [conn205876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:06:51.384Z I CONTROL [conn205876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:07:11.165Z I - [conn205930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:07:11.175Z I CONTROL [conn205930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:07:31.163Z I - [conn205457] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:07:31.172Z I CONTROL [conn205457] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:07:51.152Z I - [conn215932] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:07:51.162Z I CONTROL [conn215932] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:08:11.162Z I - [conn207416] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:08:11.171Z I CONTROL [conn207416] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:08:31.153Z I - [conn205570] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:08:31.163Z I CONTROL [conn205570] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:08:50.310Z I - [conn205681] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:08:50.320Z I CONTROL [conn205681] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:08:51.157Z I - [conn215971] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:08:51.166Z I CONTROL [conn215971] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:09:11.160Z I - [conn206775] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:09:11.170Z I CONTROL [conn206775] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:09:31.167Z I - [conn216136] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:09:31.177Z I CONTROL [conn216136] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:09:51.163Z I - [conn212599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:09:51.173Z I CONTROL [conn212599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:10:11.159Z I - [conn216112] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:10:11.169Z I CONTROL [conn216112] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:10:31.153Z I - [conn177468] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:10:31.162Z I CONTROL [conn177468] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:10:51.160Z I - [conn216156] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:10:51.170Z I CONTROL [conn216156] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:11:11.163Z I - [conn183343] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:11:11.173Z I CONTROL [conn183343] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:11:31.166Z I - [conn206261] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:11:31.176Z I CONTROL [conn206261] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:11:51.158Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:11:51.167Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:12:11.183Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:12:11.192Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:12:31.156Z I - [conn205626] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:12:31.166Z I CONTROL [conn205626] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:12:51.163Z I - [conn205917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:12:51.173Z I CONTROL [conn205917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:13:11.157Z I - [conn205499] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:13:11.167Z I CONTROL [conn205499] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:13:31.159Z I - [conn215909] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:13:31.169Z I CONTROL [conn215909] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:13:51.158Z I - [conn205798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:13:51.168Z I CONTROL [conn205798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:14:11.156Z I - [conn215977] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:14:11.166Z I CONTROL [conn215977] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:14:31.157Z I - [conn207410] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:14:31.167Z I CONTROL [conn207410] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:14:51.152Z I - [conn183088] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:14:51.162Z I CONTROL [conn183088] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:15:11.165Z I - [conn205915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:15:11.175Z I CONTROL [conn205915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:15:31.267Z I - [conn215958] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:15:31.276Z I CONTROL [conn215958] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:15:51.156Z I - [conn216104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:15:51.166Z I CONTROL [conn216104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:16:11.158Z I - [conn215139] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:16:11.168Z I CONTROL [conn215139] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:16:31.157Z I - [conn205756] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:16:31.167Z I CONTROL [conn205756] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:16:51.155Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:16:51.165Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:17:11.156Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:17:11.166Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:17:31.158Z I - [conn205666] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:17:31.168Z I CONTROL [conn205666] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:17:51.454Z I - [conn205697] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:17:51.464Z I CONTROL [conn205697] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:18:11.155Z I - [conn205690] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:18:11.165Z I CONTROL [conn205690] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:18:31.175Z I - [conn205484] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:18:31.185Z I CONTROL [conn205484] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:18:51.161Z I - [conn207410] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:18:51.170Z I CONTROL [conn207410] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:19:11.153Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:19:11.162Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:19:31.146Z I - [conn206257] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:19:31.156Z I CONTROL [conn206257] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:19:51.157Z I - [conn216062] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:19:51.167Z I CONTROL [conn216062] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:20:11.143Z I - [conn205497] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:20:11.153Z I CONTROL [conn205497] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:20:31.144Z I - [conn183103] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:20:31.154Z I CONTROL [conn183103] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:20:51.158Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:20:51.168Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:21:11.146Z I - [conn205853] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:21:11.156Z I CONTROL [conn205853] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:21:31.153Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:21:31.162Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:21:51.154Z I - [conn205599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:21:51.163Z I CONTROL [conn205599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:22:11.145Z I - [conn205675] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:22:11.154Z I CONTROL [conn205675] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:22:31.142Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:22:31.152Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:22:51.143Z I - [conn205774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:22:51.152Z I CONTROL [conn205774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:23:11.156Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:23:11.165Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:23:31.142Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:23:31.152Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:23:51.145Z I - [conn205554] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:23:51.155Z I CONTROL [conn205554] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:24:11.145Z I - [conn216105] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:24:11.154Z I CONTROL [conn216105] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:24:31.156Z I - [conn205752] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:24:31.165Z I CONTROL [conn205752] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:24:51.142Z I - [conn205949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:24:51.152Z I CONTROL [conn205949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:25:11.155Z I - [conn205664] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:25:11.165Z I CONTROL [conn205664] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:25:31.154Z I - [conn183087] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:25:31.164Z I CONTROL [conn183087] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:25:51.156Z I - [conn205664] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:25:51.166Z I CONTROL [conn205664] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:26:11.150Z I - [conn205481] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:26:11.160Z I CONTROL [conn205481] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:26:31.149Z I - [conn215923] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:26:31.159Z I CONTROL [conn215923] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:26:51.153Z I - [conn205843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:26:51.162Z I CONTROL [conn205843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:27:11.142Z I - [conn212228] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:27:11.151Z I CONTROL [conn212228] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:27:31.155Z I - [conn205554] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:27:31.164Z I CONTROL [conn205554] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:27:51.154Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:27:51.163Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:28:11.138Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:28:11.147Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:28:31.153Z I - [conn212596] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:28:31.162Z I CONTROL [conn212596] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:28:51.152Z I - [conn215858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:28:51.161Z I CONTROL [conn215858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:29:11.149Z I - [conn216160] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:29:11.158Z I CONTROL [conn216160] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:29:31.150Z I - [conn216060] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:29:31.160Z I CONTROL [conn216060] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:29:51.146Z I - [conn216095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:29:51.156Z I CONTROL [conn216095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:30:11.148Z I - [conn205644] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:30:11.158Z I CONTROL [conn205644] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:30:31.148Z I - [conn205658] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:30:31.158Z I CONTROL [conn205658] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:30:51.151Z I - [conn205577] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:30:51.161Z I CONTROL [conn205577] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:31:11.147Z I - [conn205572] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:31:11.156Z I CONTROL [conn205572] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:31:31.148Z I - [conn216066] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:31:31.158Z I CONTROL [conn216066] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:31:51.149Z I - [conn215143] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:31:51.158Z I CONTROL [conn215143] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:32:11.146Z I - [conn205931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:32:11.156Z I CONTROL [conn205931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:32:31.143Z I - [conn205478] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:32:31.153Z I CONTROL [conn205478] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:32:51.156Z I - [conn205843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:32:51.166Z I CONTROL [conn205843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:33:11.151Z I - [conn205871] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:33:11.160Z I CONTROL [conn205871] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:33:31.150Z I - [conn212228] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:33:31.159Z I CONTROL [conn212228] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:33:51.152Z I - [conn205871] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:33:51.161Z I CONTROL [conn205871] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:34:11.146Z I - [conn215941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:34:11.156Z I CONTROL [conn215941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:34:31.144Z I - [conn216141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:34:31.153Z I CONTROL [conn216141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:34:51.138Z I - [conn205664] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:34:51.147Z I CONTROL [conn205664] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:35:11.136Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:35:11.146Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:35:31.147Z I - [conn212598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:35:31.157Z I CONTROL [conn212598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:35:51.148Z I - [conn205515] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:35:51.157Z I CONTROL [conn205515] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:36:11.155Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:36:11.165Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:36:31.145Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:36:31.155Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:36:51.134Z I - [conn215939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:36:51.144Z I CONTROL [conn215939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:37:11.136Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:37:11.146Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:37:31.135Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:37:31.145Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:37:51.199Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:37:51.209Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:38:11.143Z I - [conn215927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:38:11.153Z I CONTROL [conn215927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:38:31.143Z I - [conn205874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:38:31.152Z I CONTROL [conn205874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:38:51.149Z I - [conn205880] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:38:51.158Z I CONTROL [conn205880] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:39:11.145Z I - [conn215143] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:39:11.155Z I CONTROL [conn215143] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:39:31.134Z I - [conn205442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:39:31.144Z I CONTROL [conn205442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:39:51.144Z I - [conn215144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:39:51.154Z I CONTROL [conn215144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:40:11.144Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:40:11.153Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:40:31.146Z I - [conn205931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:40:31.156Z I CONTROL [conn205931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:40:51.144Z I - [conn206304] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:40:51.153Z I CONTROL [conn206304] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:41:11.133Z I - [conn205842] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:41:11.143Z I CONTROL [conn205842] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:41:31.141Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:41:31.151Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:41:51.141Z I - [conn215950] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:41:51.150Z I CONTROL [conn215950] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:42:11.149Z I - [conn215970] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:42:11.160Z I CONTROL [conn215970] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:42:31.143Z I - [conn205808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:42:31.155Z I CONTROL [conn205808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:42:51.129Z I - [conn215913] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:42:51.139Z I CONTROL [conn215913] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:43:11.149Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:43:11.159Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:43:31.152Z I - [conn216129] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:43:31.162Z I CONTROL [conn216129] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:43:51.129Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:43:51.138Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:44:11.140Z I - [conn205843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:44:11.149Z I CONTROL [conn205843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:44:31.139Z I - [conn205639] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:44:31.148Z I CONTROL [conn205639] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:44:51.129Z I - [conn205812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:44:51.138Z I CONTROL [conn205812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:45:11.141Z I - [conn205471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:45:11.150Z I CONTROL [conn205471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:45:31.127Z I - [conn216123] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:45:31.137Z I CONTROL [conn216123] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:45:51.142Z I - [conn216060] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:45:51.152Z I CONTROL [conn216060] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:46:11.131Z I - [conn206318] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:46:11.141Z I CONTROL [conn206318] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:46:31.142Z I - [conn205798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:46:31.151Z I CONTROL [conn205798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:46:51.140Z I - [conn205736] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:46:51.150Z I CONTROL [conn205736] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:47:11.141Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:47:11.151Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:47:31.140Z I - [conn177468] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:47:31.152Z I CONTROL [conn177468] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:47:51.138Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:47:51.148Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:48:11.136Z I - [conn205722] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:48:11.146Z I CONTROL [conn205722] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:48:31.134Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:48:31.143Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:48:51.139Z I - [conn170473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:48:51.150Z I CONTROL [conn170473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:49:11.140Z I - [conn205625] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:49:11.150Z I CONTROL [conn205625] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:49:31.144Z I - [conn205842] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:49:31.154Z I CONTROL [conn205842] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:49:51.136Z I - [conn216118] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:49:51.146Z I CONTROL [conn216118] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:50:11.129Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:50:11.139Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:50:31.138Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:50:31.147Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:50:51.137Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:50:51.147Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:51:11.136Z I - [conn205743] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:51:11.146Z I CONTROL [conn205743] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:51:31.133Z I - [conn215133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:51:31.143Z I CONTROL [conn215133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:51:51.139Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:51:51.148Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:52:11.136Z I - [conn215927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:52:11.146Z I CONTROL [conn215927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:52:31.138Z I - [conn215937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:52:31.148Z I CONTROL [conn215937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:52:51.124Z I - [conn215922] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:52:51.133Z I CONTROL [conn215922] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:53:11.137Z I - [conn205548] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:53:11.147Z I CONTROL [conn205548] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:53:31.135Z I - [conn207417] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:53:31.145Z I CONTROL [conn207417] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:53:51.137Z I - [conn206331] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:53:51.146Z I CONTROL [conn206331] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:54:11.135Z I - [conn215924] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:54:11.145Z I CONTROL [conn215924] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:54:31.129Z I - [conn207413] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:54:31.138Z I CONTROL [conn207413] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:54:51.133Z I - [conn216147] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:54:51.142Z I CONTROL [conn216147] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:55:11.124Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:55:11.133Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:55:31.133Z I - [conn205737] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:55:31.143Z I CONTROL [conn205737] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:55:51.129Z I - [conn215966] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:55:51.139Z I CONTROL [conn215966] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:56:11.131Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:56:11.141Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:56:31.123Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:56:31.132Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:56:51.133Z I - [conn205837] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:56:51.143Z I CONTROL [conn205837] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:57:11.140Z I - [conn215941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:57:11.149Z I CONTROL [conn215941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:57:31.132Z I - [conn216144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:57:31.142Z I CONTROL [conn216144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:57:51.120Z I - [conn215958] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:57:51.130Z I CONTROL [conn215958] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:58:11.143Z I - [conn215944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:58:11.153Z I CONTROL [conn215944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:58:31.134Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:58:31.143Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:58:51.131Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:58:51.140Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:59:11.136Z I - [conn207402] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:59:11.145Z I CONTROL [conn207402] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:59:31.134Z I - [conn205666] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:59:31.144Z I CONTROL [conn205666] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T12:59:48.961Z I INDEX [conn215461] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T12:59:51.123Z I - [conn215938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T12:59:51.133Z I CONTROL [conn215938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:00:11.129Z I - [conn215938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:00:11.139Z I CONTROL [conn215938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:00:31.131Z I - [conn205421] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:00:31.140Z I CONTROL [conn205421] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:00:51.136Z I - [conn205796] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:00:51.146Z I CONTROL [conn205796] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:01:11.132Z I - [conn216075] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:01:11.142Z I CONTROL [conn216075] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:01:31.136Z I - [conn212601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:01:31.145Z I CONTROL [conn212601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:01:51.120Z I - [conn205586] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:01:51.130Z I CONTROL [conn205586] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:02:11.130Z I - [conn183336] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:02:11.140Z I CONTROL [conn183336] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:02:31.128Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:02:31.138Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:02:51.134Z I - [conn215855] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:02:51.143Z I CONTROL [conn215855] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:03:11.130Z I - [conn205810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:03:11.140Z I CONTROL [conn205810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:03:31.124Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:03:31.133Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:03:51.119Z I - [conn215947] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:03:51.129Z I CONTROL [conn215947] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:04:11.131Z I - [conn206311] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:04:11.141Z I CONTROL [conn206311] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:04:31.131Z I - [conn215910] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:04:31.141Z I CONTROL [conn215910] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:04:51.121Z I - [conn216115] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:04:51.131Z I CONTROL [conn216115] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:05:11.128Z I - [conn205874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:05:11.138Z I CONTROL [conn205874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:05:31.116Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:05:31.125Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:05:51.122Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:05:51.131Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:06:11.128Z I - [conn215917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:06:11.138Z I CONTROL [conn215917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:06:31.131Z I - [conn205743] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:06:31.141Z I CONTROL [conn205743] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:06:51.131Z I - [conn205487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:06:51.140Z I CONTROL [conn205487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:07:11.431Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:07:11.441Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:07:31.129Z I - [conn177440] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:07:31.138Z I CONTROL [conn177440] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:07:51.129Z I - [conn205617] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:07:51.139Z I CONTROL [conn205617] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:08:11.128Z I - [conn205625] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:08:11.139Z I CONTROL [conn205625] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:08:31.125Z I - [conn215961] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:08:31.135Z I CONTROL [conn215961] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:08:51.129Z I - [conn205935] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:08:51.138Z I CONTROL [conn205935] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:09:11.127Z I - [conn215918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:09:11.136Z I CONTROL [conn215918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:09:31.122Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:09:31.132Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:09:51.130Z I - [conn205937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:09:51.139Z I CONTROL [conn205937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:10:11.121Z I - [conn205842] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:10:11.131Z I CONTROL [conn205842] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:10:31.131Z I - [conn206798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:10:31.141Z I CONTROL [conn206798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:10:51.119Z I - [conn205849] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:10:51.128Z I CONTROL [conn205849] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:11:11.132Z I - [conn205685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:11:11.142Z I CONTROL [conn205685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:11:31.126Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:11:31.136Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:11:51.129Z I - [conn205740] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:11:51.139Z I CONTROL [conn205740] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:12:11.132Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:12:11.142Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:12:31.124Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:12:31.134Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:12:51.127Z I - [conn170504] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:12:51.137Z I CONTROL [conn170504] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:13:11.123Z I - [conn205904] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:13:11.133Z I CONTROL [conn205904] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:13:31.127Z I - [conn206811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:13:31.137Z I CONTROL [conn206811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:13:51.130Z I - [conn205500] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:13:51.139Z I CONTROL [conn205500] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:14:11.113Z I - [conn205719] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:14:11.123Z I CONTROL [conn205719] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:14:31.115Z I - [conn216136] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:14:31.125Z I CONTROL [conn216136] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:14:51.140Z I - [conn216138] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:14:51.149Z I CONTROL [conn216138] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:15:11.120Z I - [conn216102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:15:11.130Z I CONTROL [conn216102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:15:31.108Z I - [conn205840] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:15:31.117Z I CONTROL [conn205840] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:15:51.125Z I - [conn215968] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:15:51.134Z I CONTROL [conn215968] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:16:11.122Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:16:11.132Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:16:31.127Z I - [conn205933] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:16:31.136Z I CONTROL [conn205933] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:16:51.120Z I - [conn215877] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:16:51.130Z I CONTROL [conn215877] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:17:11.110Z I - [conn205481] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:17:11.120Z I CONTROL [conn205481] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:17:31.121Z I - [conn215877] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:17:31.131Z I CONTROL [conn215877] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:17:51.128Z I - [conn215144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:17:51.137Z I CONTROL [conn215144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:18:11.128Z I - [conn206811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:18:11.137Z I CONTROL [conn206811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:18:31.112Z I - [conn205664] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:18:31.122Z I CONTROL [conn205664] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:18:51.125Z I - [conn205598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:18:51.134Z I CONTROL [conn205598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:19:11.121Z I - [conn205682] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:19:11.131Z I CONTROL [conn205682] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:19:31.121Z I - [conn215144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:19:31.131Z I CONTROL [conn215144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:19:51.121Z I - [conn215914] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:19:51.130Z I CONTROL [conn215914] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:20:11.125Z I - [conn216148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:20:11.135Z I CONTROL [conn216148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:20:31.129Z I - [conn216148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:20:31.139Z I CONTROL [conn216148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:20:51.122Z I - [conn215917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:20:51.131Z I CONTROL [conn215917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:21:11.110Z I - [conn212598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:21:11.120Z I CONTROL [conn212598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:21:31.123Z I - [conn205882] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:21:31.133Z I CONTROL [conn205882] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:21:51.120Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:21:51.129Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:22:01.911Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:22:01.921Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:22:11.110Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:22:11.120Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:22:31.241Z I - [conn216097] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:22:31.251Z I CONTROL [conn216097] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:22:51.121Z I - [conn205621] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:22:51.130Z I CONTROL [conn205621] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:23:11.118Z I - [conn205628] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:23:11.128Z I CONTROL [conn205628] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:23:31.116Z I - [conn216089] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:23:31.126Z I CONTROL [conn216089] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:23:51.122Z I - [conn207405] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:23:51.132Z I CONTROL [conn207405] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:24:11.117Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:24:11.126Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:24:31.124Z I - [conn205666] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:24:31.133Z I CONTROL [conn205666] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:24:51.116Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:24:51.126Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:25:11.117Z I - [conn183171] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:25:11.127Z I CONTROL [conn183171] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:25:31.136Z I - [conn206257] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:25:31.146Z I CONTROL [conn206257] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:25:51.121Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:25:51.131Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:26:11.124Z I - [conn215963] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:26:11.133Z I CONTROL [conn215963] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:26:31.108Z I - [conn205863] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:26:31.117Z I CONTROL [conn205863] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:26:51.119Z I - [conn205863] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:26:51.129Z I CONTROL [conn205863] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:27:11.118Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:27:11.128Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:27:31.118Z I - [conn206255] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:27:31.127Z I CONTROL [conn206255] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:27:51.117Z I - [conn205872] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:27:51.126Z I CONTROL [conn205872] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:28:11.120Z I - [conn206800] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:28:11.129Z I CONTROL [conn206800] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:28:31.126Z I - [conn205814] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:28:31.135Z I CONTROL [conn205814] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:28:51.117Z I - [conn205573] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:28:51.127Z I CONTROL [conn205573] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:29:11.126Z I - [conn205814] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:29:11.136Z I CONTROL [conn205814] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:29:31.117Z I - [conn205742] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:29:31.127Z I CONTROL [conn205742] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:29:51.119Z I - [conn205945] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:29:51.129Z I CONTROL [conn205945] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:30:11.125Z I - [conn207401] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:30:11.135Z I CONTROL [conn207401] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:30:31.123Z I - [conn205917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:30:31.133Z I CONTROL [conn205917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:30:51.114Z I - [conn215922] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:30:51.124Z I CONTROL [conn215922] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:31:11.123Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:31:11.134Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:31:31.116Z I - [conn212229] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:31:31.126Z I CONTROL [conn212229] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:31:51.105Z I - [conn205867] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:31:51.114Z I CONTROL [conn205867] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:32:11.114Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:32:11.124Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:32:31.104Z I - [conn212601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:32:31.114Z I CONTROL [conn212601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:32:51.116Z I - [conn205719] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:32:51.126Z I CONTROL [conn205719] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:33:11.108Z I - [conn216095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:33:11.118Z I CONTROL [conn216095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:33:31.119Z I - [conn205456] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:33:31.129Z I CONTROL [conn205456] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:33:51.104Z I - [conn205456] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:33:51.114Z I CONTROL [conn205456] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:34:11.117Z I - [conn205949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:34:11.126Z I CONTROL [conn205949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:34:31.120Z I - [conn215918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:34:31.129Z I CONTROL [conn215918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:34:51.105Z I - [conn183095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:34:51.115Z I CONTROL [conn183095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:35:11.099Z I - [conn216060] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:35:11.109Z I CONTROL [conn216060] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:35:31.113Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:35:31.123Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:35:51.115Z I - [conn205481] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:35:51.125Z I CONTROL [conn205481] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:36:11.116Z I - [conn205615] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:36:11.126Z I CONTROL [conn205615] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:36:31.112Z I - [conn215134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:36:31.121Z I CONTROL [conn215134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:36:51.115Z I - [conn205453] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:36:51.124Z I CONTROL [conn205453] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:37:11.104Z I - [conn177379] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:37:11.113Z I CONTROL [conn177379] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:37:31.116Z I - [conn215918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:37:31.126Z I CONTROL [conn215918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:37:51.116Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:37:51.126Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:38:11.112Z I - [conn212226] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:38:11.122Z I CONTROL [conn212226] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:38:31.117Z I - [conn205528] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:38:31.127Z I CONTROL [conn205528] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:38:51.124Z I - [conn215145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:38:51.134Z I CONTROL [conn215145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:39:11.122Z I - [conn215944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:39:11.131Z I CONTROL [conn215944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:39:31.109Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:39:31.118Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:39:51.118Z I - [conn215139] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:39:51.128Z I CONTROL [conn215139] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:40:11.110Z I - [conn205478] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:40:11.120Z I CONTROL [conn205478] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:40:31.110Z I - [conn205562] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:40:31.120Z I CONTROL [conn205562] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:40:51.114Z I - [conn205598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:40:51.123Z I CONTROL [conn205598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:41:11.117Z I - [conn205920] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:41:11.127Z I CONTROL [conn205920] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:41:31.109Z I - [conn205798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:41:31.118Z I CONTROL [conn205798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:41:51.115Z I - [conn205554] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:41:51.125Z I CONTROL [conn205554] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:42:11.106Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:42:11.115Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:42:31.116Z I - [conn206811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:42:31.125Z I CONTROL [conn206811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:42:51.099Z I - [conn205791] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:42:51.109Z I CONTROL [conn205791] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:43:11.116Z I - [conn205749] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:43:11.126Z I CONTROL [conn205749] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:43:31.117Z I - [conn216128] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:43:31.127Z I CONTROL [conn216128] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:43:51.111Z I - [conn216113] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:43:51.121Z I CONTROL [conn216113] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:44:11.095Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:44:11.104Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:44:31.105Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:44:31.114Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:44:51.112Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:44:51.121Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:45:11.108Z I - [conn205609] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:45:11.118Z I CONTROL [conn205609] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:45:31.108Z I - [conn205856] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:45:31.118Z I CONTROL [conn205856] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:45:51.099Z I - [conn183429] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:45:51.109Z I CONTROL [conn183429] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:46:11.098Z I - [conn212228] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:46:11.108Z I CONTROL [conn212228] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:46:31.098Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:46:31.108Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:46:51.120Z I - [conn212599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:46:51.130Z I CONTROL [conn212599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:47:11.091Z I - [conn205942] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:47:11.100Z I CONTROL [conn205942] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:47:31.104Z I - [conn205831] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:47:31.113Z I CONTROL [conn205831] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:47:51.112Z I - [conn205562] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:47:51.122Z I CONTROL [conn205562] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:48:11.102Z I - [conn215953] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:48:11.112Z I CONTROL [conn215953] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:48:31.111Z I - [conn183336] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:48:31.121Z I CONTROL [conn183336] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:48:51.118Z I - [conn206798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:48:51.127Z I CONTROL [conn206798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:49:11.104Z I - [conn205810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:49:11.114Z I CONTROL [conn205810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:49:31.096Z I - [conn206797] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:49:31.105Z I CONTROL [conn206797] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:49:51.200Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:49:51.209Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:50:11.108Z I - [conn205634] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:50:11.118Z I CONTROL [conn205634] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:50:31.106Z I - [conn216102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:50:31.115Z I CONTROL [conn216102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:50:51.097Z I - [conn216145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:50:51.106Z I CONTROL [conn216145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:51:11.108Z I - [conn205697] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:51:11.117Z I CONTROL [conn205697] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:51:31.109Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:51:31.119Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:51:51.108Z I - [conn206255] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:51:51.118Z I CONTROL [conn206255] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:52:11.105Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:52:11.114Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:52:31.095Z I - [conn206314] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:52:31.105Z I CONTROL [conn206314] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:52:51.093Z I - [conn183114] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:52:51.103Z I CONTROL [conn183114] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:53:11.106Z I - [conn206311] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:53:11.116Z I CONTROL [conn206311] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:53:31.103Z I - [conn170471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:53:31.113Z I CONTROL [conn170471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:53:51.093Z I - [conn207402] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:53:51.103Z I CONTROL [conn207402] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:54:11.106Z I - [conn205518] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:54:11.116Z I CONTROL [conn205518] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:54:31.207Z I - [conn216156] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:54:31.217Z I CONTROL [conn216156] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:54:51.094Z I - [conn216134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:54:51.104Z I CONTROL [conn216134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:55:11.090Z I - [conn205656] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:55:11.100Z I CONTROL [conn205656] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:55:31.092Z I - [conn205935] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:55:31.102Z I CONTROL [conn205935] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:55:51.093Z I - [conn215143] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:55:51.102Z I CONTROL [conn215143] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:56:11.111Z I - [conn216110] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:56:11.121Z I CONTROL [conn216110] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:56:31.104Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:56:31.113Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:56:51.104Z I - [conn206254] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:56:51.114Z I CONTROL [conn206254] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:57:11.088Z I - [conn216146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:57:11.098Z I CONTROL [conn216146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:57:31.088Z I - [conn216124] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:57:31.097Z I CONTROL [conn216124] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:57:51.087Z I - [conn216062] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:57:51.097Z I CONTROL [conn216062] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:58:11.095Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:58:11.105Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:58:31.101Z I - [conn216155] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:58:31.111Z I CONTROL [conn216155] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:58:51.099Z I - [conn215932] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:58:51.109Z I CONTROL [conn215932] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:59:11.097Z I - [conn216125] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:59:11.107Z I CONTROL [conn216125] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:59:31.100Z I - [conn205625] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:59:31.110Z I CONTROL [conn205625] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T13:59:50.800Z I INDEX [conn215568] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T13:59:51.098Z I - [conn215968] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T13:59:51.108Z I CONTROL [conn215968] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:00:11.100Z I - [conn205548] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:00:11.109Z I CONTROL [conn205548] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:00:31.092Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:00:31.102Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:00:51.113Z I - [conn206798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:00:51.123Z I CONTROL [conn206798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:01:11.086Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:01:11.096Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:01:31.096Z I - [conn215937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:01:31.106Z I CONTROL [conn215937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:01:51.097Z I - [conn205928] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:01:51.106Z I CONTROL [conn205928] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:02:11.096Z I - [conn215973] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:02:11.105Z I CONTROL [conn215973] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:02:31.095Z I - [conn205585] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:02:31.105Z I CONTROL [conn205585] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:02:51.086Z I - [conn216127] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:02:51.096Z I CONTROL [conn216127] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:03:11.099Z I - [conn216119] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:03:11.109Z I CONTROL [conn216119] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:03:31.087Z I - [conn205609] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:03:31.096Z I CONTROL [conn205609] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:03:51.098Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:03:51.108Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:04:11.095Z I - [conn205634] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:04:11.104Z I CONTROL [conn205634] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:04:31.096Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:04:31.105Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:04:51.107Z I - [conn205928] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:04:51.116Z I CONTROL [conn205928] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:05:11.099Z I - [conn215962] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:05:11.109Z I CONTROL [conn215962] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:05:31.099Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:05:31.108Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:05:51.096Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:05:51.105Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:06:11.103Z I - [conn177404] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:06:11.113Z I CONTROL [conn177404] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:06:31.098Z I - [conn212599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:06:31.107Z I CONTROL [conn212599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:06:51.093Z I - [conn216065] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:06:51.103Z I CONTROL [conn216065] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:07:11.099Z I - [conn206316] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:07:11.109Z I CONTROL [conn206316] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:07:31.140Z I - [conn216122] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:07:31.149Z I CONTROL [conn216122] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:07:38.203Z I - [conn215941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:07:38.212Z I CONTROL [conn215941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:07:51.086Z I - [conn205856] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:07:51.095Z I CONTROL [conn205856] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:08:11.102Z I - [conn205707] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:08:11.111Z I CONTROL [conn205707] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:08:31.093Z I - [conn216146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:08:31.103Z I CONTROL [conn216146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:08:51.097Z I - [conn216071] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:08:51.106Z I CONTROL [conn216071] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:09:11.095Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:09:11.105Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:09:31.096Z I - [conn205904] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:09:31.106Z I CONTROL [conn205904] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:09:51.187Z I - [conn215983] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:09:51.197Z I CONTROL [conn215983] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:10:11.095Z I - [conn215983] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:10:11.105Z I CONTROL [conn215983] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:10:31.084Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:10:31.093Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:10:51.084Z I - [conn215929] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:10:51.094Z I CONTROL [conn215929] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:11:11.094Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:11:11.103Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:11:31.088Z I - [conn215926] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:11:31.097Z I CONTROL [conn215926] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:11:51.094Z I - [conn216129] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:11:51.104Z I CONTROL [conn216129] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:12:11.086Z I - [conn215930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:12:11.095Z I CONTROL [conn215930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:12:31.086Z I - [conn205697] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:12:31.095Z I CONTROL [conn205697] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:12:51.090Z I - [conn205876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:12:51.100Z I CONTROL [conn205876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:13:11.092Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:13:11.101Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:13:31.085Z I - [conn216124] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:13:31.095Z I CONTROL [conn216124] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:13:51.077Z I - [conn215917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:13:51.087Z I CONTROL [conn215917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:14:11.093Z I - [conn215140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:14:11.102Z I CONTROL [conn215140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:14:31.072Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:14:31.082Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:14:51.081Z I - [conn205682] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:14:51.092Z I CONTROL [conn205682] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:15:11.089Z I - [conn216123] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:15:11.099Z I CONTROL [conn216123] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:15:31.095Z I - [conn207685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:15:31.105Z I CONTROL [conn207685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:15:51.090Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:15:51.100Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:16:11.093Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:16:11.103Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:16:31.083Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:16:31.092Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:16:51.091Z I - [conn205522] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:16:51.101Z I CONTROL [conn205522] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:17:11.076Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:17:11.085Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:17:31.090Z I - [conn205871] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:17:31.100Z I CONTROL [conn205871] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:17:51.080Z I - [conn215950] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:17:51.090Z I CONTROL [conn215950] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:18:11.080Z I - [conn205573] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:18:11.090Z I CONTROL [conn205573] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:18:31.083Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:18:31.092Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:18:51.078Z I - [conn216129] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:18:51.088Z I CONTROL [conn216129] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:19:11.108Z I - [conn205499] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:19:11.118Z I CONTROL [conn205499] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:19:31.079Z I - [conn216075] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:19:31.089Z I CONTROL [conn216075] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:19:51.088Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:19:51.098Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:20:11.089Z I - [conn205915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:20:11.098Z I CONTROL [conn205915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:20:31.089Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:20:31.099Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:20:51.089Z I - [conn206798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:20:51.099Z I CONTROL [conn206798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:21:11.088Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:21:11.097Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:21:31.088Z I - [conn205604] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:21:31.097Z I CONTROL [conn205604] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:21:51.089Z I - [conn215874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:21:51.099Z I CONTROL [conn215874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:22:11.072Z I - [conn216131] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:22:11.081Z I CONTROL [conn216131] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:22:31.087Z I - [conn205787] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:22:31.096Z I CONTROL [conn205787] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:22:51.077Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:22:51.087Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:23:11.087Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:23:11.097Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:23:31.086Z I - [conn205878] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:23:31.096Z I CONTROL [conn205878] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:23:51.074Z I - [conn216145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:23:51.083Z I CONTROL [conn216145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:24:11.086Z I - [conn215910] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:24:11.095Z I CONTROL [conn215910] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:24:31.089Z I - [conn183429] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:24:31.098Z I CONTROL [conn183429] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:24:51.076Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:24:51.086Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:25:11.085Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:25:11.096Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:25:31.087Z I - [conn205625] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:25:31.096Z I CONTROL [conn205625] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:25:51.083Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:25:51.094Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:26:11.080Z I - [conn206771] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:26:11.090Z I CONTROL [conn206771] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:26:31.085Z I - [conn205570] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:26:31.095Z I CONTROL [conn205570] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:26:51.076Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:26:51.086Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:27:11.086Z I - [conn205728] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:27:11.096Z I CONTROL [conn205728] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:27:31.087Z I - [conn205708] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:27:31.096Z I CONTROL [conn205708] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:27:51.087Z I - [conn206313] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:27:51.096Z I CONTROL [conn206313] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:28:11.077Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:28:11.087Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:28:31.081Z I - [conn216099] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:28:31.091Z I CONTROL [conn216099] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:28:51.087Z I - [conn177440] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:28:51.097Z I CONTROL [conn177440] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:29:11.113Z I - [conn206774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:29:11.123Z I CONTROL [conn206774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:29:31.087Z I - [conn215974] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:29:31.097Z I CONTROL [conn215974] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:29:51.084Z I - [conn183429] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:29:51.094Z I CONTROL [conn183429] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:30:11.077Z I - [conn215966] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:30:11.086Z I CONTROL [conn215966] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:30:31.071Z I - [conn179981] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:30:31.080Z I CONTROL [conn179981] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:30:51.065Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:30:51.075Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:31:11.083Z I - [conn207410] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:31:11.093Z I CONTROL [conn207410] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:31:31.080Z I - [conn216094] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:31:31.090Z I CONTROL [conn216094] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:31:51.088Z I - [conn205457] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:31:51.098Z I CONTROL [conn205457] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:32:11.081Z I - [conn216137] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:32:11.090Z I CONTROL [conn216137] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:32:31.082Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:32:31.092Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:32:51.098Z I - [conn205478] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:32:51.107Z I CONTROL [conn205478] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:33:11.087Z I - [conn215961] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:33:11.097Z I CONTROL [conn215961] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:33:31.083Z I - [conn205908] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:33:31.093Z I CONTROL [conn205908] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:33:51.081Z I - [conn215974] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:33:51.091Z I CONTROL [conn215974] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:34:11.083Z I - [conn212225] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:34:11.092Z I CONTROL [conn212225] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:34:31.082Z I - [conn205878] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:34:31.091Z I CONTROL [conn205878] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:34:51.068Z I - [conn205908] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:34:51.078Z I CONTROL [conn205908] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:35:11.084Z I - [conn205941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:35:11.094Z I CONTROL [conn205941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:35:31.071Z I - [conn216097] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:35:31.080Z I CONTROL [conn216097] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:35:51.069Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:35:51.079Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:36:11.074Z I - [conn205858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:36:11.084Z I CONTROL [conn205858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:36:31.080Z I - [conn205949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:36:31.090Z I CONTROL [conn205949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:36:51.078Z I - [conn205714] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:36:51.088Z I CONTROL [conn205714] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:37:11.083Z I - [conn206332] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:37:11.093Z I CONTROL [conn206332] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:37:31.078Z I - [conn205945] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:37:31.088Z I CONTROL [conn205945] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:37:51.081Z I - [conn206797] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:37:51.091Z I CONTROL [conn206797] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:38:11.081Z I - [conn216137] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:38:11.091Z I CONTROL [conn216137] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:38:31.080Z I - [conn205816] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:38:31.090Z I CONTROL [conn205816] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:38:51.066Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:38:51.075Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:39:11.077Z I - [conn216104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:39:11.087Z I CONTROL [conn216104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:39:31.070Z I - [conn179981] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:39:31.079Z I CONTROL [conn179981] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:39:51.075Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:39:51.084Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:40:11.077Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:40:11.086Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:40:31.079Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:40:31.089Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:40:51.071Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:40:51.081Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:41:11.077Z I - [conn205453] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:41:11.086Z I CONTROL [conn205453] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:41:31.074Z I - [conn205453] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:41:31.084Z I CONTROL [conn205453] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:41:51.080Z I - [conn179981] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:41:51.089Z I CONTROL [conn179981] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:42:11.083Z I - [conn205945] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:42:11.093Z I CONTROL [conn205945] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:42:31.076Z I - [conn205675] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:42:31.086Z I CONTROL [conn205675] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:42:51.089Z I - [conn183095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:42:51.099Z I CONTROL [conn183095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:43:11.075Z I - [conn215953] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:43:11.085Z I CONTROL [conn215953] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:43:31.074Z I - [conn183095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:43:31.084Z I CONTROL [conn183095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:43:51.065Z I - [conn216153] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:43:51.074Z I CONTROL [conn216153] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:44:11.080Z I - [conn215859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:44:11.090Z I CONTROL [conn215859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:44:31.081Z I - [conn205756] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:44:31.091Z I CONTROL [conn205756] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:44:51.085Z I - [conn205837] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:44:51.094Z I CONTROL [conn205837] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:45:11.078Z I - [conn205813] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:45:11.087Z I CONTROL [conn205813] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:45:31.064Z I - [conn206328] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:45:31.074Z I CONTROL [conn206328] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:45:51.076Z I - [conn205572] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:45:51.085Z I CONTROL [conn205572] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:46:11.073Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:46:11.082Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:46:31.057Z I - [conn215926] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:46:31.066Z I CONTROL [conn215926] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:46:51.072Z I - [conn215880] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:46:51.082Z I CONTROL [conn215880] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:47:11.077Z I - [conn205471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:47:11.087Z I CONTROL [conn205471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:47:31.065Z I - [conn215133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:47:31.075Z I CONTROL [conn215133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:47:51.078Z I - [conn177378] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:47:51.087Z I CONTROL [conn177378] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:48:11.069Z I - [conn215918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:48:11.079Z I CONTROL [conn215918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:48:31.072Z I - [conn205692] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:48:31.081Z I CONTROL [conn205692] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:48:51.078Z I - [conn205465] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:48:51.088Z I CONTROL [conn205465] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:49:11.061Z I - [conn205692] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:49:11.070Z I CONTROL [conn205692] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:49:31.073Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:49:31.083Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:49:51.071Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:49:51.080Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:50:11.073Z I - [conn205824] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:50:11.083Z I CONTROL [conn205824] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:50:31.066Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:50:31.076Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:50:51.075Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:50:51.085Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:51:11.072Z I - [conn205457] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:51:11.082Z I CONTROL [conn205457] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:51:18.046Z I - [conn205639] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:51:18.056Z I CONTROL [conn205639] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:51:31.069Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:51:31.079Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:51:51.072Z I - [conn205876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:51:51.082Z I CONTROL [conn205876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:52:11.072Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:52:11.081Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:52:31.073Z I - [conn205880] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:52:31.083Z I CONTROL [conn205880] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:52:51.070Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:52:51.080Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:53:11.069Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:53:11.079Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:53:31.071Z I - [conn206260] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:53:31.081Z I CONTROL [conn206260] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:53:51.082Z I - [conn207414] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:53:51.091Z I CONTROL [conn207414] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:54:11.059Z I - [conn205791] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:54:11.068Z I CONTROL [conn205791] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:54:31.075Z I - [conn212229] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:54:31.085Z I CONTROL [conn212229] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:54:51.056Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:54:51.065Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:55:11.064Z I - [conn205880] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:55:11.074Z I CONTROL [conn205880] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:55:31.061Z I - [conn216079] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:55:31.071Z I CONTROL [conn216079] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:55:51.060Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:55:51.070Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:56:11.071Z I - [conn205691] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:56:11.081Z I CONTROL [conn205691] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:56:31.074Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:56:31.084Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:56:51.071Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:56:51.081Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:57:11.059Z I - [conn183336] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:57:11.068Z I CONTROL [conn183336] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:57:31.071Z I - [conn183088] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:57:31.081Z I CONTROL [conn183088] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:57:51.069Z I - [conn205568] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:57:51.078Z I CONTROL [conn205568] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:58:11.064Z I - [conn216135] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:58:11.074Z I CONTROL [conn216135] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:58:31.070Z I - [conn207685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:58:31.080Z I CONTROL [conn207685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:58:51.065Z I - [conn205863] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:58:51.074Z I CONTROL [conn205863] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:59:11.069Z I - [conn216155] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:59:11.079Z I CONTROL [conn216155] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:59:31.062Z I - [conn215963] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:59:31.071Z I CONTROL [conn215963] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T14:59:49.427Z I INDEX [conn215568] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T14:59:51.070Z I - [conn205604] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T14:59:51.080Z I CONTROL [conn205604] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:00:11.068Z I - [conn216099] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:00:11.078Z I CONTROL [conn216099] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:00:31.066Z I - [conn206311] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:00:31.075Z I CONTROL [conn206311] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:00:51.068Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:00:51.077Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:01:11.071Z I - [conn205824] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:01:11.080Z I CONTROL [conn205824] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:01:31.070Z I - [conn205554] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:01:31.080Z I CONTROL [conn205554] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:01:51.049Z I - [conn215958] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:01:51.059Z I CONTROL [conn215958] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:02:11.056Z I - [conn216078] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:02:11.066Z I CONTROL [conn216078] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:02:31.072Z I - [conn207412] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:02:31.081Z I CONTROL [conn207412] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:02:51.067Z I - [conn183343] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:02:51.077Z I CONTROL [conn183343] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:03:11.068Z I - [conn177419] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:03:11.079Z I CONTROL [conn177419] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:03:31.056Z I - [conn205500] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:03:31.065Z I CONTROL [conn205500] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:03:51.075Z I - [conn205783] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:03:51.085Z I CONTROL [conn205783] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:04:11.070Z I - [conn207401] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:04:11.079Z I CONTROL [conn207401] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:04:31.073Z I - [conn205465] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:04:31.083Z I CONTROL [conn205465] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:04:51.070Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:04:51.079Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:05:11.058Z I - [conn205756] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:05:11.067Z I CONTROL [conn205756] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:05:31.068Z I - [conn216104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:05:31.077Z I CONTROL [conn216104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:05:51.062Z I - [conn215979] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:05:51.071Z I CONTROL [conn215979] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:06:11.058Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:06:11.068Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:06:31.061Z I - [conn205570] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:06:31.071Z I CONTROL [conn205570] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:06:51.057Z I - [conn205465] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:06:51.066Z I CONTROL [conn205465] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:07:11.067Z I - [conn170473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:07:11.077Z I CONTROL [conn170473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:07:31.062Z I - [conn205633] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:07:31.071Z I CONTROL [conn205633] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:07:51.066Z I - [conn216111] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:07:51.076Z I CONTROL [conn216111] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:08:11.088Z I - [conn212229] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:08:11.097Z I CONTROL [conn212229] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:08:31.066Z I - [conn216111] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:08:31.075Z I CONTROL [conn216111] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:08:51.064Z I - [conn216074] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:08:51.074Z I CONTROL [conn216074] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:09:11.057Z I - [conn177440] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:09:11.067Z I CONTROL [conn177440] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:09:31.064Z I - [conn205857] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:09:31.074Z I CONTROL [conn205857] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:09:51.058Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:09:51.067Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:10:11.054Z I - [conn215874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:10:11.065Z I CONTROL [conn215874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:10:31.055Z I - [conn212230] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:10:31.065Z I CONTROL [conn212230] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:10:51.057Z I - [conn215937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:10:51.066Z I CONTROL [conn215937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:11:11.064Z I - [conn216135] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:11:11.075Z I CONTROL [conn216135] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:11:31.053Z I - [conn216071] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:11:31.064Z I CONTROL [conn216071] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:11:51.064Z I - [conn206317] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:11:51.073Z I CONTROL [conn206317] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:12:11.055Z I - [conn178530] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:12:11.065Z I CONTROL [conn178530] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:12:31.055Z I - [conn215960] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:12:31.067Z I CONTROL [conn215960] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:12:51.051Z I - [conn216131] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:12:51.060Z I CONTROL [conn216131] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:13:11.058Z I - [conn207408] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:13:11.068Z I CONTROL [conn207408] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:13:31.045Z I - [conn216131] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:13:31.055Z I CONTROL [conn216131] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:13:51.224Z I - [conn215977] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:13:51.234Z I CONTROL [conn215977] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:14:11.062Z I - [conn171685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:14:11.072Z I CONTROL [conn171685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:14:31.053Z I - [conn206329] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:14:31.063Z I CONTROL [conn206329] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:14:51.066Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:14:51.076Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:15:11.065Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:15:11.075Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:15:31.063Z I - [conn205837] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:15:31.072Z I CONTROL [conn205837] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:15:51.061Z I - [conn215876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:15:51.070Z I CONTROL [conn215876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:16:11.052Z I - [conn205837] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:16:11.061Z I CONTROL [conn205837] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:16:31.072Z I - [conn205808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:16:31.081Z I CONTROL [conn205808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:16:51.052Z I - [conn207410] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:16:51.062Z I CONTROL [conn207410] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:17:11.064Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:17:11.073Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:17:31.051Z I - [conn215917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:17:31.061Z I CONTROL [conn215917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:17:51.060Z I - [conn216071] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:17:51.070Z I CONTROL [conn216071] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:18:11.055Z I - [conn205878] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:18:11.065Z I CONTROL [conn205878] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:18:31.064Z I - [conn206316] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:18:31.074Z I CONTROL [conn206316] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:18:51.058Z I - [conn205634] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:18:51.067Z I CONTROL [conn205634] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:19:11.061Z I - [conn216071] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:19:11.071Z I CONTROL [conn216071] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:19:31.060Z I - [conn215947] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:19:31.070Z I CONTROL [conn215947] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:19:51.062Z I - [conn183104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:19:51.072Z I CONTROL [conn183104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:20:11.067Z I - [conn212599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:20:11.077Z I CONTROL [conn212599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:20:31.051Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:20:31.061Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:20:51.051Z I - [conn212599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:20:51.061Z I CONTROL [conn212599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:21:11.060Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:21:11.069Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:21:31.047Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:21:31.056Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:21:51.062Z I - [conn215932] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:21:51.072Z I CONTROL [conn215932] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:22:11.066Z I - [conn205774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:22:11.076Z I CONTROL [conn205774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:22:31.069Z I - [conn205726] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:22:31.079Z I CONTROL [conn205726] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:22:51.062Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:22:51.072Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:23:11.058Z I - [conn216132] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:23:11.067Z I CONTROL [conn216132] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:23:31.061Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:23:31.070Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:23:51.050Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:23:51.060Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:24:11.048Z I - [conn215960] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:24:11.058Z I CONTROL [conn215960] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:24:31.059Z I - [conn205572] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:24:31.069Z I CONTROL [conn205572] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:24:51.061Z I - [conn205682] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:24:51.073Z I CONTROL [conn205682] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:25:11.058Z I - [conn205479] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:25:11.067Z I CONTROL [conn205479] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:25:31.066Z I - [conn205518] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:25:31.076Z I CONTROL [conn205518] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:25:51.065Z I - [conn206807] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:25:51.075Z I CONTROL [conn206807] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:26:11.048Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:26:11.058Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:26:31.047Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:26:31.057Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:26:51.062Z I - [conn206797] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:26:51.072Z I CONTROL [conn206797] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:27:11.060Z I - [conn216060] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:27:11.069Z I CONTROL [conn216060] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:27:31.058Z I - [conn205880] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:27:31.068Z I CONTROL [conn205880] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:27:51.059Z I - [conn205479] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:27:51.069Z I CONTROL [conn205479] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:28:11.058Z I - [conn205942] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:28:11.067Z I CONTROL [conn205942] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:28:31.064Z I - [conn177379] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:28:31.074Z I CONTROL [conn177379] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:28:51.056Z I - [conn215957] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:28:51.066Z I CONTROL [conn215957] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:29:11.056Z I - [conn205756] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:29:11.065Z I CONTROL [conn205756] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:29:31.036Z I - [conn205941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:29:31.046Z I CONTROL [conn205941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:29:51.057Z I - [conn207417] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:29:51.067Z I CONTROL [conn207417] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:30:11.051Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:30:11.061Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:30:31.045Z I - [conn216155] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:30:31.055Z I CONTROL [conn216155] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:30:51.056Z I - [conn205487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:30:51.066Z I CONTROL [conn205487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:31:11.043Z I - [conn212601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:31:11.052Z I CONTROL [conn212601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:31:31.049Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:31:31.059Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:31:51.044Z I - [conn215921] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:31:51.054Z I CONTROL [conn215921] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:32:11.053Z I - [conn206261] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:32:11.062Z I CONTROL [conn206261] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:32:31.037Z I - [conn205719] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:32:31.047Z I CONTROL [conn205719] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:32:51.048Z I - [conn205719] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:32:51.058Z I CONTROL [conn205719] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:33:11.046Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:33:11.056Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:33:31.047Z I - [conn205736] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:33:31.056Z I CONTROL [conn205736] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:33:51.037Z I - [conn205831] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:33:51.046Z I CONTROL [conn205831] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:34:11.056Z I - [conn215909] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:34:11.066Z I CONTROL [conn215909] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:34:31.047Z I - [conn177440] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:34:31.057Z I CONTROL [conn177440] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:34:51.045Z I - [conn205685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:34:51.055Z I CONTROL [conn205685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:35:11.047Z I - [conn206317] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:35:11.057Z I CONTROL [conn206317] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:35:31.046Z I - [conn206317] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:35:31.055Z I CONTROL [conn206317] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:35:51.040Z I - [conn215939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:35:51.050Z I CONTROL [conn215939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:36:11.053Z I - [conn216157] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:36:11.063Z I CONTROL [conn216157] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:36:31.045Z I - [conn205948] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:36:31.054Z I CONTROL [conn205948] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:36:51.046Z I - [conn206796] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:36:51.056Z I CONTROL [conn206796] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:37:11.047Z I - [conn215145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:37:11.057Z I CONTROL [conn215145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:37:31.043Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:37:31.052Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:37:51.058Z I - [conn183343] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:37:51.068Z I CONTROL [conn183343] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:38:11.040Z I - [conn205456] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:38:11.050Z I CONTROL [conn205456] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:38:31.045Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:38:31.054Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:38:51.057Z I - [conn215926] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:38:51.067Z I CONTROL [conn215926] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:39:11.046Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:39:11.056Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:39:31.051Z I - [conn206314] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:39:31.061Z I CONTROL [conn206314] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:39:51.041Z I - [conn216099] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:39:51.051Z I CONTROL [conn216099] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:40:11.049Z I - [conn215938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:40:11.059Z I CONTROL [conn215938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:40:31.038Z I - [conn205810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:40:31.048Z I CONTROL [conn205810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:40:51.050Z I - [conn205484] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:40:51.060Z I CONTROL [conn205484] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:41:11.042Z I - [conn205547] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:41:11.051Z I CONTROL [conn205547] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:41:31.036Z I - [conn205548] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:41:31.046Z I CONTROL [conn205548] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:41:51.036Z I - [conn216082] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:41:51.046Z I CONTROL [conn216082] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:42:11.040Z I - [conn215927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:42:11.050Z I CONTROL [conn215927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:42:31.045Z I - [conn205804] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:42:31.055Z I CONTROL [conn205804] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:42:51.051Z I - [conn216122] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:42:51.060Z I CONTROL [conn216122] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:43:11.048Z I - [conn206775] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:43:11.057Z I CONTROL [conn206775] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:43:31.040Z I - [conn215922] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:43:31.050Z I CONTROL [conn215922] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:43:51.040Z I - [conn205831] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:43:51.049Z I CONTROL [conn205831] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:44:11.036Z I - [conn216110] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:44:11.046Z I CONTROL [conn216110] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:44:31.039Z I - [conn205473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:44:31.049Z I CONTROL [conn205473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:44:51.039Z I - [conn205908] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:44:51.049Z I CONTROL [conn205908] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:45:11.053Z I - [conn205479] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:45:11.063Z I CONTROL [conn205479] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:45:31.037Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:45:31.046Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:45:51.041Z I - [conn205765] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:45:51.050Z I CONTROL [conn205765] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:46:11.042Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:46:11.051Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:46:31.042Z I - [conn206319] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:46:31.052Z I CONTROL [conn206319] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:46:51.038Z I - [conn206314] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:46:51.048Z I CONTROL [conn206314] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:47:11.038Z I - [conn215965] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:47:11.047Z I CONTROL [conn215965] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:47:31.057Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:47:31.066Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:47:51.050Z I - [conn206798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:47:51.059Z I CONTROL [conn206798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:48:11.053Z I - [conn205749] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:48:11.063Z I CONTROL [conn205749] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:48:31.027Z I - [conn183344] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:48:31.037Z I CONTROL [conn183344] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:48:51.037Z I - [conn205765] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:48:51.047Z I CONTROL [conn205765] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:49:11.040Z I - [conn178530] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:49:11.053Z I CONTROL [conn178530] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:49:31.051Z I - [conn205853] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:49:31.061Z I CONTROL [conn205853] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:49:51.038Z I - [conn215849] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:49:51.048Z I CONTROL [conn215849] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:50:11.053Z I - [conn207408] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:50:11.063Z I CONTROL [conn207408] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:50:31.062Z I - [conn216119] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:50:31.072Z I CONTROL [conn216119] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:50:51.053Z I - [conn215140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:50:51.063Z I CONTROL [conn215140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:51:11.046Z I - [conn205500] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:51:11.056Z I CONTROL [conn205500] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:51:31.044Z I - [conn205621] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:51:31.054Z I CONTROL [conn205621] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:51:51.026Z I - [conn205824] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:51:51.035Z I CONTROL [conn205824] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:52:11.037Z I - [conn205933] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:52:11.047Z I CONTROL [conn205933] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:52:31.036Z I - [conn205933] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:52:31.045Z I CONTROL [conn205933] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:52:51.026Z I - [conn205933] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:52:51.035Z I CONTROL [conn205933] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:53:11.026Z I - [conn215930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:53:11.036Z I CONTROL [conn215930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:53:31.039Z I - [conn205900] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:53:31.049Z I CONTROL [conn205900] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:53:51.042Z I - [conn215978] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:53:51.052Z I CONTROL [conn215978] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:54:11.035Z I - [conn205813] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:54:11.045Z I CONTROL [conn205813] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:54:31.043Z I - [conn216148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:54:31.053Z I CONTROL [conn216148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:54:51.030Z I - [conn216137] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:54:51.040Z I CONTROL [conn216137] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:55:11.034Z I - [conn205572] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:55:11.044Z I CONTROL [conn205572] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:55:31.044Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:55:31.054Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:55:51.028Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:55:51.038Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:56:11.038Z I - [conn205878] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:56:11.047Z I CONTROL [conn205878] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:56:31.038Z I - [conn205664] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:56:31.048Z I CONTROL [conn205664] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:56:51.037Z I - [conn205577] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:56:51.047Z I CONTROL [conn205577] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:57:11.032Z I - [conn216091] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:57:11.042Z I CONTROL [conn216091] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:57:31.031Z I - [conn216091] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:57:31.040Z I CONTROL [conn216091] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:57:51.036Z I - [conn205599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:57:51.046Z I CONTROL [conn205599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:58:11.036Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:58:11.047Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:58:31.046Z I - [conn205908] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:58:31.055Z I CONTROL [conn205908] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:58:51.033Z I - [conn205801] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:58:51.044Z I CONTROL [conn205801] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:59:11.030Z I - [conn216137] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:59:11.040Z I CONTROL [conn216137] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:59:31.030Z I - [conn205848] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:59:31.039Z I CONTROL [conn205848] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T15:59:49.787Z I INDEX [conn215568] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T15:59:51.035Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T15:59:51.045Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:00:11.033Z I - [conn205554] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:00:11.042Z I CONTROL [conn205554] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:00:31.033Z I - [conn205626] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:00:31.045Z I CONTROL [conn205626] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:00:51.031Z I - [conn205815] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:00:51.040Z I CONTROL [conn205815] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:01:11.034Z I - [conn177419] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:01:11.043Z I CONTROL [conn177419] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:01:31.042Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:01:31.052Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:01:51.042Z I - [conn170504] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:01:51.052Z I CONTROL [conn170504] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:02:11.044Z I - [conn216091] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:02:11.053Z I CONTROL [conn216091] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:02:31.029Z I - [conn183336] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:02:31.039Z I CONTROL [conn183336] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:02:51.033Z I - [conn215918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:02:51.043Z I CONTROL [conn215918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:03:11.048Z I - [conn207402] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:03:11.058Z I CONTROL [conn207402] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:03:31.041Z I - [conn205743] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:03:31.051Z I CONTROL [conn205743] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:03:51.039Z I - [conn216060] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:03:51.049Z I CONTROL [conn216060] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:04:11.032Z I - [conn183336] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:04:11.041Z I CONTROL [conn183336] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:04:31.029Z I - [conn205626] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:04:31.039Z I CONTROL [conn205626] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:04:51.025Z I - [conn216072] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:04:51.034Z I CONTROL [conn216072] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:05:11.027Z I - [conn216072] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:05:11.037Z I CONTROL [conn216072] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:05:31.028Z I - [conn216115] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:05:31.037Z I CONTROL [conn216115] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:05:51.016Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:05:51.026Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:06:11.032Z I - [conn206775] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:06:11.042Z I CONTROL [conn206775] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:06:31.029Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:06:31.039Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:06:37.657Z I - [conn215855] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:06:37.666Z I CONTROL [conn215855] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:06:51.026Z I - [conn205473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:06:51.036Z I CONTROL [conn205473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:07:11.030Z I - [conn215960] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:07:11.039Z I CONTROL [conn215960] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:07:31.039Z I - [conn205776] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:07:31.049Z I CONTROL [conn205776] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:07:51.015Z I - [conn216078] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:07:51.024Z I CONTROL [conn216078] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:08:11.030Z I - [conn206331] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:08:11.039Z I CONTROL [conn206331] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:08:31.030Z I - [conn215880] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:08:31.040Z I CONTROL [conn215880] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:08:51.040Z I - [conn215935] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:08:51.050Z I CONTROL [conn215935] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:09:11.030Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:09:11.040Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:09:31.037Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:09:31.046Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:09:51.025Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:09:51.034Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:10:11.023Z I - [conn215962] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:10:11.033Z I CONTROL [conn215962] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:10:31.046Z I - [conn183088] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:10:31.056Z I CONTROL [conn183088] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:10:51.027Z I - [conn216148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:10:51.037Z I CONTROL [conn216148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:11:11.034Z I - [conn205933] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:11:11.044Z I CONTROL [conn205933] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:11:31.031Z I - [conn205621] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:11:31.040Z I CONTROL [conn205621] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:11:51.030Z I - [conn183336] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:11:51.040Z I CONTROL [conn183336] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:12:11.027Z I - [conn205691] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:12:11.037Z I CONTROL [conn205691] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:12:31.024Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:12:31.035Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:12:51.022Z I - [conn215914] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:12:51.032Z I CONTROL [conn215914] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:13:11.024Z I - [conn216141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:13:11.034Z I CONTROL [conn216141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:13:31.034Z I - [conn216141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:13:31.044Z I CONTROL [conn216141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:13:51.027Z I - [conn206332] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:13:51.037Z I CONTROL [conn206332] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:14:11.027Z I - [conn216073] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:14:11.036Z I CONTROL [conn216073] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:14:31.024Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:14:31.034Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:14:51.022Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:14:51.032Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:15:11.013Z I - [conn216118] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:15:11.023Z I CONTROL [conn216118] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:15:31.025Z I - [conn205837] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:15:31.034Z I CONTROL [conn205837] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:15:51.026Z I - [conn215979] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:15:51.037Z I CONTROL [conn215979] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:16:11.020Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:16:11.030Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:16:31.020Z I - [conn216141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:16:31.030Z I CONTROL [conn216141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:16:51.021Z I - [conn205930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:16:51.031Z I CONTROL [conn205930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:17:11.023Z I - [conn206332] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:17:11.032Z I CONTROL [conn206332] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:17:31.022Z I - [conn215874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:17:31.031Z I CONTROL [conn215874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:17:51.026Z I - [conn205456] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:17:51.036Z I CONTROL [conn205456] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:18:11.023Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:18:11.033Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:18:31.023Z I - [conn216156] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:18:31.032Z I CONTROL [conn216156] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:18:51.186Z I - [conn215877] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:18:51.195Z I CONTROL [conn215877] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:19:11.769Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:19:11.779Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:19:31.012Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:19:31.022Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:19:39.814Z I - [conn215914] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:19:39.824Z I CONTROL [conn215914] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:19:51.025Z I - [conn205515] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:19:51.034Z I CONTROL [conn205515] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:20:11.020Z I - [conn215931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:20:11.030Z I CONTROL [conn215931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:20:31.020Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:20:31.029Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:20:51.035Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:20:51.045Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:21:11.021Z I - [conn216124] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:21:11.030Z I CONTROL [conn216124] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:21:31.016Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:21:31.025Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:21:51.026Z I - [conn205683] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:21:51.036Z I CONTROL [conn205683] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:22:11.022Z I - [conn205783] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:22:11.032Z I CONTROL [conn205783] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:22:31.024Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:22:31.033Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:22:51.020Z I - [conn216102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:22:51.030Z I CONTROL [conn216102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:23:11.024Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:23:11.034Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:23:31.024Z I - [conn205871] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:23:31.034Z I CONTROL [conn205871] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:23:51.022Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:23:51.031Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:24:11.021Z I - [conn216158] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:24:11.031Z I CONTROL [conn216158] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:24:31.021Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:24:31.030Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:24:51.020Z I - [conn207412] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:24:51.030Z I CONTROL [conn207412] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:25:11.031Z I - [conn205933] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:25:11.040Z I CONTROL [conn205933] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:25:31.020Z I - [conn216078] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:25:31.029Z I CONTROL [conn216078] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:25:51.021Z I - [conn205904] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:25:51.030Z I CONTROL [conn205904] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:26:11.014Z I - [conn212601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:26:11.024Z I CONTROL [conn212601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:26:31.023Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:26:31.033Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:26:51.020Z I - [conn205682] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:26:51.030Z I CONTROL [conn205682] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:27:11.018Z I - [conn205787] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:27:11.028Z I CONTROL [conn205787] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:27:31.013Z I - [conn205634] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:27:31.023Z I CONTROL [conn205634] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:27:51.017Z I - [conn216146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:27:51.026Z I CONTROL [conn216146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:28:11.023Z I - [conn205573] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:28:11.033Z I CONTROL [conn205573] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:28:31.019Z I - [conn205621] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:28:31.028Z I CONTROL [conn205621] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:28:51.020Z I - [conn205783] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:28:51.029Z I CONTROL [conn205783] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:29:11.011Z I - [conn205812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:29:11.020Z I CONTROL [conn205812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:29:31.011Z I - [conn170504] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:29:31.021Z I CONTROL [conn170504] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:29:51.011Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:29:51.021Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:30:11.017Z I - [conn215927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:30:11.027Z I CONTROL [conn215927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:30:31.022Z I - [conn206807] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:30:31.032Z I CONTROL [conn206807] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:30:51.017Z I - [conn205863] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:30:51.026Z I CONTROL [conn205863] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:31:11.008Z I - [conn215958] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:31:11.018Z I CONTROL [conn215958] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:31:31.016Z I - [conn206318] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:31:31.026Z I CONTROL [conn206318] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:31:51.013Z I - [conn216155] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:31:51.023Z I CONTROL [conn216155] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:32:11.017Z I - [conn183114] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:32:11.026Z I CONTROL [conn183114] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:32:16.298Z I - [conn216068] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:32:16.308Z I CONTROL [conn216068] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:32:31.025Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:32:31.035Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:32:51.025Z I - [conn215913] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:32:51.035Z I CONTROL [conn215913] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:33:11.016Z I - [conn205818] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:33:11.026Z I CONTROL [conn205818] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:33:31.012Z I - [conn216062] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:33:31.022Z I CONTROL [conn216062] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:33:51.048Z I - [conn216080] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:33:51.059Z I CONTROL [conn216080] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:34:11.011Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:34:11.021Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:34:31.020Z I - [conn205690] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:34:31.030Z I CONTROL [conn205690] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:34:51.014Z I - [conn205707] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:34:51.024Z I CONTROL [conn205707] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:35:11.013Z I - [conn206317] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:35:11.022Z I CONTROL [conn206317] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:35:31.024Z I - [conn215937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:35:31.034Z I CONTROL [conn215937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:35:51.016Z I - [conn215931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:35:51.025Z I CONTROL [conn215931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:36:11.012Z I - [conn216153] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:36:11.022Z I CONTROL [conn216153] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:36:31.038Z I - [conn179981] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:36:31.048Z I CONTROL [conn179981] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:36:51.016Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:36:51.026Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:37:11.014Z I - [conn205658] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:37:11.024Z I CONTROL [conn205658] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:37:31.011Z I - [conn170812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:37:31.021Z I CONTROL [conn170812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:37:51.015Z I - [conn206798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:37:51.025Z I CONTROL [conn206798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:38:11.015Z I - [conn205486] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:38:11.024Z I CONTROL [conn205486] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:38:31.025Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:38:31.036Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:38:31.992Z I - [conn206774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:38:32.002Z I CONTROL [conn206774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:38:51.016Z I - [conn207410] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:38:51.025Z I CONTROL [conn207410] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:39:11.013Z I - [conn170487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:39:11.023Z I CONTROL [conn170487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:39:31.022Z I - [conn205682] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:39:31.031Z I CONTROL [conn205682] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:39:51.017Z I - [conn205843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:39:51.026Z I CONTROL [conn205843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:40:11.014Z I - [conn206796] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:40:11.024Z I CONTROL [conn206796] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:40:31.026Z I - [conn212599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:40:31.036Z I CONTROL [conn212599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:40:51.022Z I - [conn216110] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:40:51.031Z I CONTROL [conn216110] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:41:11.011Z I - [conn205765] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:41:11.020Z I CONTROL [conn205765] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:41:31.011Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:41:31.021Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:41:51.013Z I - [conn205629] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:41:51.023Z I CONTROL [conn205629] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:42:11.012Z I - [conn205644] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:42:11.022Z I CONTROL [conn205644] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:42:31.012Z I - [conn170898] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:42:31.022Z I CONTROL [conn170898] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:42:51.010Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:42:51.020Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:43:11.013Z I - [conn216122] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:43:11.023Z I CONTROL [conn216122] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:43:31.011Z I - [conn205882] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:43:31.021Z I CONTROL [conn205882] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:43:51.012Z I - [conn205471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:43:51.021Z I CONTROL [conn205471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:44:11.008Z I - [conn170504] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:44:11.017Z I CONTROL [conn170504] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:44:31.010Z I - [conn205499] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:44:31.020Z I CONTROL [conn205499] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:44:51.017Z I - [conn215913] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:44:51.026Z I CONTROL [conn215913] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:45:11.010Z I - [conn205808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:45:11.020Z I CONTROL [conn205808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:45:14.441Z I - [conn216117] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:45:14.450Z I CONTROL [conn216117] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:45:14.569Z I - [conn215956] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:45:14.579Z I CONTROL [conn215956] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:45:31.009Z I - [conn205816] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:45:31.018Z I CONTROL [conn205816] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:45:51.010Z I - [conn216117] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:45:51.019Z I CONTROL [conn216117] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:46:11.022Z I - [conn183095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:46:11.032Z I CONTROL [conn183095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:46:31.004Z I - [conn205842] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:46:31.014Z I CONTROL [conn205842] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:46:51.030Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:46:51.039Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:47:11.012Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:47:11.021Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:47:31.003Z I - [conn205740] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:47:31.013Z I CONTROL [conn205740] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:47:51.012Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:47:51.021Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:48:11.015Z I - [conn216117] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:48:11.024Z I CONTROL [conn216117] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:48:12.078Z I - [conn205765] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:48:12.088Z I CONTROL [conn205765] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:48:31.007Z I - [conn205842] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:48:31.016Z I CONTROL [conn205842] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:48:51.012Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:48:51.022Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:49:11.005Z I - [conn215939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:49:11.015Z I CONTROL [conn215939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:49:31.016Z I - [conn205644] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:49:31.025Z I CONTROL [conn205644] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:49:51.007Z I - [conn205749] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:49:51.017Z I CONTROL [conn205749] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:50:11.005Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:50:11.015Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:50:31.008Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:50:31.018Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:50:51.007Z I - [conn215939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:50:51.016Z I CONTROL [conn215939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:51:11.005Z I - [conn216155] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:51:11.015Z I CONTROL [conn216155] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:51:31.003Z I - [conn205572] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:51:31.012Z I CONTROL [conn205572] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:51:51.008Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:51:51.017Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:52:10.999Z I - [conn207401] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:52:11.009Z I CONTROL [conn207401] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:52:31.052Z I - [conn215909] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:52:31.061Z I CONTROL [conn215909] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:52:51.016Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:52:51.025Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:53:11.016Z I - [conn216095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:53:11.025Z I CONTROL [conn216095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:53:31.007Z I - [conn215139] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:53:31.017Z I CONTROL [conn215139] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:53:50.996Z I - [conn215913] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:53:51.006Z I CONTROL [conn215913] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:54:11.015Z I - [conn205810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:54:11.024Z I CONTROL [conn205810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:54:31.016Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:54:31.025Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:54:51.009Z I - [conn212228] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:54:51.019Z I CONTROL [conn212228] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:55:11.004Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:55:11.015Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:55:31.006Z I - [conn216118] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:55:31.016Z I CONTROL [conn216118] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:55:51.011Z I - [conn205604] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:55:51.021Z I CONTROL [conn205604] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:56:11.006Z I - [conn205465] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:56:11.016Z I CONTROL [conn205465] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:56:31.004Z I - [conn215854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:56:31.013Z I CONTROL [conn215854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:56:51.002Z I - [conn215922] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:56:51.012Z I CONTROL [conn215922] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:57:11.002Z I - [conn215983] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:57:11.011Z I CONTROL [conn215983] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:57:30.998Z I - [conn216122] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:57:31.008Z I CONTROL [conn216122] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:57:51.011Z I - [conn205728] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:57:51.021Z I CONTROL [conn205728] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:58:11.002Z I - [conn205421] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:58:11.012Z I CONTROL [conn205421] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:58:31.006Z I - [conn206328] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:58:31.015Z I CONTROL [conn206328] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:58:51.014Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:58:51.024Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:59:11.007Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:59:11.017Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:59:30.996Z I - [conn216149] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:59:31.005Z I CONTROL [conn216149] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T16:59:49.698Z I INDEX [conn215513] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T16:59:51.016Z I - [conn216110] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T16:59:51.025Z I CONTROL [conn216110] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:00:11.005Z I - [conn206813] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:00:11.014Z I CONTROL [conn206813] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:00:31.006Z I - [conn212226] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:00:31.016Z I CONTROL [conn212226] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:00:51.034Z I - [conn205586] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:00:51.044Z I CONTROL [conn205586] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:01:10.994Z I - [conn205479] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:01:11.004Z I CONTROL [conn205479] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:01:31.001Z I - [conn205481] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:01:31.010Z I CONTROL [conn205481] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:01:51.004Z I - [conn215877] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:01:51.014Z I CONTROL [conn215877] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:02:11.006Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:02:11.015Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:02:31.003Z I - [conn205783] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:02:31.013Z I CONTROL [conn205783] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:02:50.994Z I - [conn205633] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:02:51.003Z I CONTROL [conn205633] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:03:11.004Z I - [conn215977] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:03:11.014Z I CONTROL [conn215977] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:03:31.013Z I - [conn205810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:03:31.023Z I CONTROL [conn205810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:03:51.002Z I - [conn215962] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:03:51.011Z I CONTROL [conn215962] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:04:11.008Z I - [conn207408] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:04:11.017Z I CONTROL [conn207408] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:04:30.999Z I - [conn215956] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:04:31.008Z I CONTROL [conn215956] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:04:51.004Z I - [conn206813] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:04:51.014Z I CONTROL [conn206813] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:05:11.024Z I - [conn205756] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:05:11.033Z I CONTROL [conn205756] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:05:30.998Z I - [conn215956] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:05:31.008Z I CONTROL [conn215956] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:05:51.015Z I - [conn206261] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:05:51.025Z I CONTROL [conn206261] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:06:11.011Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:06:11.021Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:06:31.001Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:06:31.010Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:06:51.005Z I - [conn205842] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:06:51.015Z I CONTROL [conn205842] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:07:11.005Z I - [conn207410] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:07:11.015Z I CONTROL [conn207410] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:07:31.009Z I - [conn215914] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:07:31.019Z I CONTROL [conn215914] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:07:51.002Z I - [conn205916] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:07:51.012Z I CONTROL [conn205916] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:08:11.022Z I - [conn205811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:08:11.032Z I CONTROL [conn205811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:08:31.000Z I - [conn216099] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:08:31.010Z I CONTROL [conn216099] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:08:50.999Z I - [conn205900] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:08:51.009Z I CONTROL [conn205900] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:09:11.005Z I - [conn212226] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:09:11.014Z I CONTROL [conn212226] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:09:30.998Z I - [conn205867] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:09:31.007Z I CONTROL [conn205867] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:09:51.001Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:09:51.010Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:10:11.001Z I - [conn206811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:10:11.010Z I CONTROL [conn206811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:10:31.000Z I - [conn215979] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:10:31.010Z I CONTROL [conn215979] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:10:50.997Z I - [conn177440] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:10:51.006Z I CONTROL [conn177440] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:11:11.000Z I - [conn206774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:11:11.009Z I CONTROL [conn206774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:11:30.992Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:11:31.001Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:11:51.004Z I - [conn183087] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:11:51.014Z I CONTROL [conn183087] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:12:10.995Z I - [conn206320] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:12:11.005Z I CONTROL [conn206320] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:12:30.995Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:12:31.004Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:12:50.995Z I - [conn215962] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:12:51.005Z I CONTROL [conn215962] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:13:10.998Z I - [conn215911] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:13:11.008Z I CONTROL [conn215911] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:13:31.000Z I - [conn205737] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:13:31.010Z I CONTROL [conn205737] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:13:51.001Z I - [conn206797] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:13:51.011Z I CONTROL [conn206797] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:14:10.996Z I - [conn183087] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:14:11.006Z I CONTROL [conn183087] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:14:31.016Z I - [conn205783] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:14:31.026Z I CONTROL [conn205783] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:14:51.001Z I - [conn207411] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:14:51.010Z I CONTROL [conn207411] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:15:11.119Z I - [conn206331] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:15:11.129Z I CONTROL [conn206331] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:15:30.998Z I - [conn215850] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:15:31.008Z I CONTROL [conn215850] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:15:51.016Z I - [conn205421] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:15:51.026Z I CONTROL [conn205421] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:16:11.005Z I - [conn215924] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:16:11.015Z I CONTROL [conn215924] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:16:30.988Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:16:30.997Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:16:51.008Z I - [conn205743] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:16:51.017Z I CONTROL [conn205743] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:17:11.000Z I - [conn206768] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:17:11.010Z I CONTROL [conn206768] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:17:30.988Z I - [conn205479] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:17:30.998Z I CONTROL [conn205479] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:17:50.993Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:17:51.002Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:18:11.005Z I - [conn205815] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:18:11.014Z I CONTROL [conn205815] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:18:30.989Z I - [conn205690] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:18:30.998Z I CONTROL [conn205690] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:18:50.987Z I - [conn205783] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:18:50.997Z I CONTROL [conn205783] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:19:10.993Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:19:11.003Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:19:30.995Z I - [conn205737] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:19:31.004Z I CONTROL [conn205737] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:19:50.991Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:19:51.001Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:20:10.984Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:20:10.994Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:20:30.998Z I - [conn205457] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:20:31.007Z I CONTROL [conn205457] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:20:50.994Z I - [conn205749] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:20:51.003Z I CONTROL [conn205749] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:21:11.006Z I - [conn207411] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:21:11.016Z I CONTROL [conn207411] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:21:30.990Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:21:31.000Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:21:50.996Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:21:51.006Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:22:10.994Z I - [conn205749] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:22:11.004Z I CONTROL [conn205749] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:22:30.995Z I - [conn205421] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:22:31.004Z I CONTROL [conn205421] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:22:50.995Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:22:51.005Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:23:10.995Z I - [conn216069] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:23:11.005Z I CONTROL [conn216069] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:23:30.996Z I - [conn177404] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:23:31.006Z I CONTROL [conn177404] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:23:43.088Z I - [conn216092] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:23:43.097Z I CONTROL [conn216092] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:23:50.987Z I - [conn205471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:23:50.997Z I CONTROL [conn205471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:24:10.992Z I - [conn205756] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:24:11.002Z I CONTROL [conn205756] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:24:31.008Z I - [conn207411] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:24:31.018Z I CONTROL [conn207411] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:24:50.995Z I - [conn205714] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:24:51.004Z I CONTROL [conn205714] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:25:10.996Z I - [conn205863] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:25:11.005Z I CONTROL [conn205863] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:25:31.006Z I - [conn215935] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:25:31.015Z I CONTROL [conn215935] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:25:50.990Z I - [conn216102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:25:50.999Z I CONTROL [conn216102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:26:10.991Z I - [conn205481] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:26:11.000Z I CONTROL [conn205481] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:26:30.988Z I - [conn215927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:26:30.998Z I CONTROL [conn215927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:26:50.992Z I - [conn206261] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:26:51.001Z I CONTROL [conn206261] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:27:10.991Z I - [conn205644] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:27:11.000Z I CONTROL [conn205644] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:27:30.992Z I - [conn205482] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:27:31.002Z I CONTROL [conn205482] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:27:50.995Z I - [conn205882] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:27:51.004Z I CONTROL [conn205882] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:28:11.012Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:28:11.022Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:28:30.981Z I - [conn207414] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:28:30.991Z I CONTROL [conn207414] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:28:51.010Z I - [conn205935] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:28:51.020Z I CONTROL [conn205935] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:29:10.989Z I - [conn212599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:29:10.999Z I CONTROL [conn212599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:29:30.992Z I - [conn216099] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:29:31.001Z I CONTROL [conn216099] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:29:51.012Z I - [conn205918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:29:51.022Z I CONTROL [conn205918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:30:10.980Z I - [conn216134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:30:10.990Z I CONTROL [conn216134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:30:30.986Z I - [conn215922] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:30:30.996Z I CONTROL [conn215922] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:30:50.985Z I - [conn216148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:30:50.995Z I CONTROL [conn216148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:31:10.987Z I - [conn183088] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:31:10.997Z I CONTROL [conn183088] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:31:31.000Z I - [conn205570] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:31:31.009Z I CONTROL [conn205570] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:31:50.990Z I - [conn205570] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:31:51.000Z I CONTROL [conn205570] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:32:10.987Z I - [conn215909] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:32:10.996Z I CONTROL [conn215909] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:32:30.989Z I - [conn216071] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:32:30.998Z I CONTROL [conn216071] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:32:50.994Z I - [conn215914] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:32:51.004Z I CONTROL [conn215914] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:33:11.350Z I - [conn215877] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:33:11.360Z I CONTROL [conn215877] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:33:30.990Z I - [conn205928] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:33:30.999Z I CONTROL [conn205928] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:33:50.978Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:33:50.988Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:34:10.986Z I - [conn215938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:34:10.996Z I CONTROL [conn215938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:34:30.980Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:34:30.989Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:34:50.987Z I - [conn215859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:34:50.996Z I CONTROL [conn215859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:35:10.984Z I - [conn205867] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:35:10.994Z I CONTROL [conn205867] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:35:31.018Z I - [conn205880] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:35:31.028Z I CONTROL [conn205880] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:35:50.989Z I - [conn205740] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:35:50.999Z I CONTROL [conn205740] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:36:10.986Z I - [conn216156] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:36:10.996Z I CONTROL [conn216156] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:36:30.989Z I - [conn205568] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:36:30.999Z I CONTROL [conn205568] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:36:50.994Z I - [conn205471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:36:51.004Z I CONTROL [conn205471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:37:10.987Z I - [conn215144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:37:10.997Z I CONTROL [conn215144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:37:30.987Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:37:30.996Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:37:50.989Z I - [conn205685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:37:50.998Z I CONTROL [conn205685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:38:10.990Z I - [conn183114] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:38:11.000Z I CONTROL [conn183114] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:38:30.984Z I - [conn216115] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:38:30.993Z I CONTROL [conn216115] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:38:51.001Z I - [conn205774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:38:51.011Z I CONTROL [conn205774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:39:10.973Z I - [conn216080] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:39:10.982Z I CONTROL [conn216080] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:39:30.982Z I - [conn215935] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:39:30.992Z I CONTROL [conn215935] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:39:50.990Z I - [conn205867] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:39:51.000Z I CONTROL [conn205867] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:40:10.980Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:40:10.990Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:40:30.978Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:40:30.987Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:40:50.989Z I - [conn205783] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:40:50.998Z I CONTROL [conn205783] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:41:10.988Z I - [conn205726] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:41:10.997Z I CONTROL [conn205726] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:41:30.979Z I - [conn205728] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:41:30.988Z I CONTROL [conn205728] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:41:50.981Z I - [conn215880] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:41:50.990Z I CONTROL [conn215880] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:42:10.981Z I - [conn205829] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:42:10.991Z I CONTROL [conn205829] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:42:30.986Z I - [conn216134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:42:30.995Z I CONTROL [conn216134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:42:51.001Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:42:51.011Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:43:10.999Z I - [conn205810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:43:11.009Z I CONTROL [conn205810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:43:31.002Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:43:31.011Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:43:50.983Z I - [conn215145] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:43:50.993Z I CONTROL [conn215145] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:44:10.991Z I - [conn216111] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:44:11.000Z I CONTROL [conn216111] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:44:30.981Z I - [conn205804] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:44:30.991Z I CONTROL [conn205804] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:44:50.973Z I - [conn215953] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:44:50.983Z I CONTROL [conn215953] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:45:10.977Z I - [conn216156] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:45:10.986Z I CONTROL [conn216156] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:45:30.972Z I - [conn183115] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:45:30.981Z I CONTROL [conn183115] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:45:50.983Z I - [conn206312] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:45:50.992Z I CONTROL [conn206312] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:46:10.980Z I - [conn206320] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:46:10.989Z I CONTROL [conn206320] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:46:30.981Z I - [conn205943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:46:30.992Z I CONTROL [conn205943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:46:50.976Z I - [conn215970] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:46:50.986Z I CONTROL [conn215970] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:47:10.980Z I - [conn205840] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:47:10.989Z I CONTROL [conn205840] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:47:30.977Z I - [conn215970] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:47:30.986Z I CONTROL [conn215970] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:47:50.979Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:47:50.988Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:48:10.977Z I - [conn205573] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:48:10.987Z I CONTROL [conn205573] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:48:30.980Z I - [conn205573] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:48:30.990Z I CONTROL [conn205573] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:48:50.984Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:48:50.994Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:49:10.978Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:49:10.987Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:49:30.978Z I - [conn216111] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:49:30.987Z I CONTROL [conn216111] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:49:50.968Z I - [conn216155] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:49:50.977Z I CONTROL [conn216155] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:50:10.972Z I - [conn205639] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:50:10.982Z I CONTROL [conn205639] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:50:30.987Z I - [conn215979] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:50:30.997Z I CONTROL [conn215979] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:50:47.539Z I - [conn215755] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:50:47.548Z I CONTROL [conn215755] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:50:50.985Z I - [conn205858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:50:50.994Z I CONTROL [conn205858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:51:10.975Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:51:10.985Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:51:30.983Z I - [conn177378] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:51:30.993Z I CONTROL [conn177378] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:51:50.978Z I - [conn205916] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:51:50.988Z I CONTROL [conn205916] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:52:10.986Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:52:10.995Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:52:30.978Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:52:30.988Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:52:50.970Z I - [conn170898] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:52:50.979Z I CONTROL [conn170898] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:53:10.982Z I - [conn179981] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:53:10.992Z I CONTROL [conn179981] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:53:30.976Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:53:30.986Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:53:50.967Z I - [conn183095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:53:50.977Z I CONTROL [conn183095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:54:10.974Z I - [conn183095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:54:10.984Z I CONTROL [conn183095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:54:30.977Z I - [conn205916] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:54:30.986Z I CONTROL [conn205916] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:54:50.978Z I - [conn215140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:54:50.988Z I CONTROL [conn215140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:55:10.969Z I - [conn205916] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:55:10.978Z I CONTROL [conn205916] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:55:30.995Z I - [conn205621] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:55:31.005Z I CONTROL [conn205621] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:55:50.975Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:55:50.984Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:56:10.965Z I - [conn183088] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:56:10.975Z I CONTROL [conn183088] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:56:30.974Z I - [conn215877] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:56:30.984Z I CONTROL [conn215877] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:56:50.964Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:56:50.973Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:57:10.978Z I - [conn205706] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:57:10.989Z I CONTROL [conn205706] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:57:30.973Z I - [conn206797] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:57:30.982Z I CONTROL [conn206797] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:57:50.976Z I - [conn205749] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:57:50.985Z I CONTROL [conn205749] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:58:10.979Z I - [conn216072] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:58:10.988Z I CONTROL [conn216072] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:58:30.972Z I - [conn205811] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:58:30.982Z I CONTROL [conn205811] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:58:51.254Z I - [conn205617] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:58:51.264Z I CONTROL [conn205617] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:59:10.976Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:59:10.986Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:59:30.975Z I - [conn205931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:59:30.988Z I CONTROL [conn205931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T17:59:48.868Z I INDEX [conn215535] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T17:59:51.348Z I - [conn215943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T17:59:51.358Z I CONTROL [conn215943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:00:12.559Z I - [conn205944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:00:12.569Z I CONTROL [conn205944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:00:30.964Z I - [conn205804] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:00:30.977Z I CONTROL [conn205804] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:00:50.962Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:00:50.976Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:01:10.990Z I - [conn216119] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:01:11.000Z I CONTROL [conn216119] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:01:31.593Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:01:31.604Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:01:51.356Z I - [conn205944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:01:51.366Z I CONTROL [conn205944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:02:10.989Z I - [conn215963] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:02:11.000Z I CONTROL [conn215963] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:02:33.352Z I - [conn205453] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:02:33.362Z I CONTROL [conn205453] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:02:54.165Z I - [conn206316] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:02:54.175Z I CONTROL [conn206316] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:03:14.137Z I - [conn215937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:03:14.147Z I CONTROL [conn215937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:03:34.074Z I - [conn206331] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:03:34.085Z I CONTROL [conn206331] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:03:50.965Z I - [conn215966] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:03:50.980Z I CONTROL [conn215966] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:04:10.966Z I - [conn216078] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:04:10.976Z I CONTROL [conn216078] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:04:30.998Z I - [conn215941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:04:31.008Z I CONTROL [conn215941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:04:50.971Z I - [conn215937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:04:50.981Z I CONTROL [conn215937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:05:10.972Z I - [conn205465] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:05:10.982Z I CONTROL [conn205465] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:05:30.964Z I - [conn205609] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:05:30.973Z I CONTROL [conn205609] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:05:50.967Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:05:50.977Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:06:10.984Z I - [conn205460] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:06:10.993Z I CONTROL [conn205460] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:06:30.972Z I - [conn216078] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:06:30.981Z I CONTROL [conn216078] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:06:50.975Z I - [conn216065] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:06:50.985Z I CONTROL [conn216065] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:07:10.971Z I - [conn216065] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:07:10.981Z I CONTROL [conn216065] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:07:30.974Z I - [conn205937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:07:30.983Z I CONTROL [conn205937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:07:42.487Z I - [conn216161] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:07:42.497Z I CONTROL [conn216161] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:07:50.965Z I - [conn215852] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:07:50.980Z I CONTROL [conn215852] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:08:10.992Z I - [conn205917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:08:11.003Z I CONTROL [conn205917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:08:30.985Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:08:30.995Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:08:50.976Z I - [conn215875] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:08:51.003Z I CONTROL [conn215875] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:09:10.973Z I - [conn215875] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:09:10.982Z I CONTROL [conn215875] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:09:30.965Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:09:30.974Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:09:50.968Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:09:50.978Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:10:10.957Z I - [conn206797] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:10:10.967Z I CONTROL [conn206797] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:10:30.968Z I - [conn177419] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:10:30.978Z I CONTROL [conn177419] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:10:50.967Z I - [conn215977] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:10:50.977Z I CONTROL [conn215977] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:11:10.965Z I - [conn205518] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:11:10.975Z I CONTROL [conn205518] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:11:30.984Z I - [conn216134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:11:30.994Z I CONTROL [conn216134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:11:50.972Z I - [conn205928] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:11:50.982Z I CONTROL [conn205928] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:12:10.955Z I - [conn206317] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:12:10.965Z I CONTROL [conn206317] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:12:30.959Z I - [conn206797] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:12:30.969Z I CONTROL [conn206797] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:12:50.969Z I - [conn205604] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:12:50.978Z I CONTROL [conn205604] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:13:10.963Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:13:10.973Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:13:30.963Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:13:30.972Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:13:50.979Z I - [conn206318] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:13:50.989Z I CONTROL [conn206318] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:14:10.964Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:14:10.973Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:14:30.960Z I - [conn215851] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:14:30.970Z I CONTROL [conn215851] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:14:50.964Z I - [conn215943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:14:50.973Z I CONTROL [conn215943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:15:10.962Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:15:10.972Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:15:30.964Z I - [conn205818] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:15:30.974Z I CONTROL [conn205818] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:15:50.954Z I - [conn205714] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:15:50.963Z I CONTROL [conn205714] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:16:10.983Z I - [conn206314] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:16:10.992Z I CONTROL [conn206314] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:16:30.990Z I - [conn206797] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:16:30.999Z I CONTROL [conn206797] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:16:50.949Z I - [conn216076] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:16:50.959Z I CONTROL [conn216076] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:17:10.989Z I - [conn205935] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:17:10.999Z I CONTROL [conn205935] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:17:30.965Z I - [conn216076] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:17:30.974Z I CONTROL [conn216076] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:17:50.954Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:17:50.963Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:18:10.961Z I - [conn205630] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:18:10.971Z I CONTROL [conn205630] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:18:30.954Z I - [conn205856] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:18:30.963Z I CONTROL [conn205856] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:18:50.964Z I - [conn205630] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:18:50.973Z I CONTROL [conn205630] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:19:10.980Z I - [conn212601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:19:10.989Z I CONTROL [conn212601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:19:30.955Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:19:30.964Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:19:50.959Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:19:50.968Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:20:10.960Z I - [conn205915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:20:10.969Z I CONTROL [conn205915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:20:30.948Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:20:30.957Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:20:50.956Z I - [conn216104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:20:50.965Z I CONTROL [conn216104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:21:10.952Z I - [conn205931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:21:10.961Z I CONTROL [conn205931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:21:30.960Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:21:30.969Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:21:50.961Z I - [conn215977] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:21:50.970Z I CONTROL [conn215977] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:22:10.953Z I - [conn212601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:22:10.963Z I CONTROL [conn212601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:22:30.963Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:22:30.972Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:22:50.959Z I - [conn215941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:22:50.969Z I CONTROL [conn215941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:23:10.970Z I - [conn205831] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:23:10.980Z I CONTROL [conn205831] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:23:30.963Z I - [conn216076] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:23:30.973Z I CONTROL [conn216076] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:23:50.958Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:23:50.967Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:24:10.968Z I - [conn170473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:24:10.978Z I CONTROL [conn170473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:24:30.958Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:24:30.968Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:24:50.960Z I - [conn183346] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:24:50.972Z I CONTROL [conn183346] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:25:10.948Z I - [conn205948] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:25:10.957Z I CONTROL [conn205948] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:25:30.955Z I - [conn215941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:25:30.964Z I CONTROL [conn215941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:25:50.959Z I - [conn215947] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:25:50.969Z I CONTROL [conn215947] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:26:10.985Z I - [conn212601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:26:10.995Z I CONTROL [conn212601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:26:30.953Z I - [conn216151] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:26:30.963Z I CONTROL [conn216151] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:26:50.961Z I - [conn205683] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:26:50.971Z I CONTROL [conn205683] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:27:10.959Z I - [conn183103] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:27:10.968Z I CONTROL [conn183103] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:27:30.956Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:27:30.966Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:27:50.960Z I - [conn205822] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:27:50.970Z I CONTROL [conn205822] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:28:10.987Z I - [conn216134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:28:10.997Z I CONTROL [conn216134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:28:30.954Z I - [conn216137] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:28:30.964Z I CONTROL [conn216137] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:28:50.954Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:28:50.963Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:29:10.954Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:29:10.964Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:29:30.944Z I - [conn215975] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:29:30.954Z I CONTROL [conn215975] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:29:50.984Z I - [conn215148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:29:50.994Z I CONTROL [conn215148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:30:10.951Z I - [conn205599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:30:10.960Z I CONTROL [conn205599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:30:30.959Z I - [conn216144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:30:30.969Z I CONTROL [conn216144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:30:50.979Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:30:50.989Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:31:10.961Z I - [conn216076] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:31:10.971Z I CONTROL [conn216076] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:35:44.636Z I - [conn216122] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:35:44.646Z I CONTROL [conn216122] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:36:02.951Z I - [conn205877] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:36:02.960Z I CONTROL [conn205877] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:36:22.942Z I - [conn212599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:36:22.952Z I CONTROL [conn212599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:36:42.933Z I - [conn205499] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:36:42.943Z I CONTROL [conn205499] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:37:02.937Z I - [conn216080] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:37:02.947Z I CONTROL [conn216080] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:37:22.931Z I - [conn216080] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:37:22.940Z I CONTROL [conn216080] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:37:42.942Z I - [conn205858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:37:42.952Z I CONTROL [conn205858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:38:02.947Z I - [conn216144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:38:02.957Z I CONTROL [conn216144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:38:22.952Z I - [conn215138] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:38:22.962Z I CONTROL [conn215138] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:38:42.952Z I - [conn170473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:38:42.961Z I CONTROL [conn170473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:39:02.942Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:39:02.952Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:39:22.940Z I - [conn205826] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:39:22.950Z I CONTROL [conn205826] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:39:42.938Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:39:42.947Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:40:02.953Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:40:02.963Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:40:22.941Z I - [conn205478] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:40:22.950Z I CONTROL [conn205478] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:40:42.938Z I - [conn216110] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:40:42.948Z I CONTROL [conn216110] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:41:02.930Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:41:02.940Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:41:22.951Z I - [conn206316] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:41:22.961Z I CONTROL [conn206316] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:41:42.944Z I - [conn205658] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:41:42.953Z I CONTROL [conn205658] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:42:02.945Z I - [conn212599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:42:02.954Z I CONTROL [conn212599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:42:22.929Z I - [conn206315] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:42:22.939Z I CONTROL [conn206315] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:42:42.930Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:42:42.939Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:43:02.951Z I - [conn215856] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:43:02.961Z I CONTROL [conn215856] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:43:22.935Z I - [conn205573] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:43:22.945Z I CONTROL [conn205573] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:43:42.952Z I - [conn177467] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:43:42.961Z I CONTROL [conn177467] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:44:02.938Z I - [conn183099] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:44:02.947Z I CONTROL [conn183099] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:44:22.938Z I - [conn170812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:44:22.947Z I CONTROL [conn170812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:44:42.944Z I - [conn206809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:44:42.954Z I CONTROL [conn206809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:45:02.934Z I - [conn215971] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:45:02.943Z I CONTROL [conn215971] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:45:22.942Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:45:22.951Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:45:42.928Z I - [conn206794] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:45:42.938Z I CONTROL [conn206794] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:46:02.929Z I - [conn205931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:46:02.939Z I CONTROL [conn205931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:46:22.938Z I - [conn205752] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:46:22.948Z I CONTROL [conn205752] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:46:42.939Z I - [conn216066] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:46:42.948Z I CONTROL [conn216066] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:47:02.929Z I - [conn216066] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:47:02.939Z I CONTROL [conn216066] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:47:22.943Z I - [conn212227] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:47:22.953Z I CONTROL [conn212227] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:47:42.947Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:47:42.957Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:48:02.935Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:48:02.944Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:48:22.937Z I - [conn205487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:48:22.946Z I CONTROL [conn205487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:48:42.937Z I - [conn205487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:48:42.946Z I CONTROL [conn205487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:49:02.944Z I - [conn216137] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:49:02.953Z I CONTROL [conn216137] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:49:22.937Z I - [conn183095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:49:22.947Z I CONTROL [conn183095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:49:42.938Z I - [conn215149] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:49:42.947Z I CONTROL [conn215149] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:50:02.927Z I - [conn215910] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:50:02.936Z I CONTROL [conn215910] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:50:22.952Z I - [conn205566] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:50:22.962Z I CONTROL [conn205566] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:50:42.926Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:50:42.936Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:51:02.934Z I - [conn177440] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:51:02.944Z I CONTROL [conn177440] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:51:22.931Z I - [conn216080] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:51:22.941Z I CONTROL [conn216080] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:51:42.932Z I - [conn206260] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:51:42.941Z I CONTROL [conn206260] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:52:02.942Z I - [conn216146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:52:02.952Z I CONTROL [conn216146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:52:22.937Z I - [conn205691] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:52:22.946Z I CONTROL [conn205691] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:52:42.936Z I - [conn215851] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:52:42.945Z I CONTROL [conn215851] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:53:02.924Z I - [conn205554] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:53:02.933Z I CONTROL [conn205554] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:53:22.942Z I - [conn216112] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:53:22.952Z I CONTROL [conn216112] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:53:42.931Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:53:42.940Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:54:02.923Z I - [conn216078] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:54:02.932Z I CONTROL [conn216078] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:54:22.933Z I - [conn171685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:54:22.943Z I CONTROL [conn171685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:54:42.929Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:54:42.939Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:55:02.929Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:55:02.939Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:55:22.934Z I - [conn205831] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:55:22.943Z I CONTROL [conn205831] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:55:42.941Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:55:42.951Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:56:02.932Z I - [conn205917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:56:02.942Z I CONTROL [conn205917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:56:22.260Z I - [conn223242] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:56:22.267Z I CONTROL [conn223242] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:56:22.268Z I QUERY [conn223242] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-28T18:56:22.929Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:56:22.939Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:56:42.930Z I - [conn215926] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:56:42.940Z I CONTROL [conn215926] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:57:02.932Z I - [conn216123] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:57:02.941Z I CONTROL [conn216123] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:57:22.939Z I - [conn216123] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:57:22.949Z I CONTROL [conn216123] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:57:42.931Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:57:42.941Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:58:02.923Z I - [conn183095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:58:02.933Z I CONTROL [conn183095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:58:22.932Z I - [conn205708] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:58:22.942Z I CONTROL [conn205708] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:58:37.767Z I - [conn223242] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:58:37.774Z I CONTROL [conn223242] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:58:37.774Z I QUERY [conn223242] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-28T18:58:42.923Z I - [conn205714] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:58:42.932Z I CONTROL [conn205714] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:59:02.932Z I - [conn206317] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:59:02.941Z I CONTROL [conn206317] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:59:22.922Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:59:22.931Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:59:42.931Z I - [conn212599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T18:59:42.941Z I CONTROL [conn212599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T18:59:51.354Z I INDEX [conn215512] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T19:00:02.926Z I - [conn205708] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:00:02.936Z I CONTROL [conn205708] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:00:22.928Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:00:22.938Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:00:42.921Z I - [conn177467] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:00:42.931Z I CONTROL [conn177467] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:01:02.927Z I - [conn216122] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:01:02.937Z I CONTROL [conn216122] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:01:22.941Z I - [conn170812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:01:22.950Z I CONTROL [conn170812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:01:42.930Z I - [conn170473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:01:42.940Z I CONTROL [conn170473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:02:02.941Z I - [conn212599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:02:02.950Z I CONTROL [conn212599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:02:22.923Z I - [conn216100] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:02:22.933Z I CONTROL [conn216100] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:02:42.936Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:02:42.946Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:03:02.921Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:03:02.931Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:03:22.930Z I - [conn205804] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:03:22.940Z I CONTROL [conn205804] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:03:42.937Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:03:42.947Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:04:02.936Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:04:02.946Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:04:22.925Z I - [conn205930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:04:22.935Z I CONTROL [conn205930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:04:42.937Z I - [conn215851] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:04:42.946Z I CONTROL [conn215851] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:05:02.926Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:05:02.941Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:05:22.930Z I - [conn205646] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:05:22.939Z I CONTROL [conn205646] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:05:42.923Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:05:42.932Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:06:02.925Z I - [conn215958] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:06:02.935Z I CONTROL [conn215958] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:06:22.916Z I - [conn205944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:06:22.927Z I CONTROL [conn205944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:06:42.946Z I - [conn205944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:06:42.956Z I CONTROL [conn205944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:07:02.923Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:07:02.933Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:07:22.917Z I - [conn170473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:07:22.927Z I CONTROL [conn170473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:07:42.938Z I - [conn216066] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:07:42.948Z I CONTROL [conn216066] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:08:02.924Z I - [conn205928] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:08:02.933Z I CONTROL [conn205928] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:08:22.913Z I - [conn216096] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:08:22.923Z I CONTROL [conn216096] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:08:42.925Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:08:42.935Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:09:02.923Z I - [conn205682] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:09:02.932Z I CONTROL [conn205682] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:09:22.926Z I - [conn205843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:09:22.937Z I CONTROL [conn205843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:09:42.929Z I - [conn205917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:09:42.939Z I CONTROL [conn205917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:10:02.931Z I - [conn206314] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:10:02.941Z I CONTROL [conn206314] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:10:22.935Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:10:22.945Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:10:42.916Z I - [conn177468] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:10:42.925Z I CONTROL [conn177468] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:11:02.924Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:11:02.933Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:11:22.927Z I - [conn205843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:11:22.936Z I CONTROL [conn205843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:11:42.933Z I - [conn216053] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:11:42.943Z I CONTROL [conn216053] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:12:02.934Z I - [conn215876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:12:02.943Z I CONTROL [conn215876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:12:22.915Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:12:22.924Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:12:42.924Z I - [conn216126] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:12:42.935Z I CONTROL [conn216126] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:13:02.924Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:13:02.936Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:13:22.945Z I - [conn205787] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:13:22.954Z I CONTROL [conn205787] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:13:42.933Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:13:42.943Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:14:02.925Z I - [conn205628] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:14:02.934Z I CONTROL [conn205628] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:14:22.925Z I - [conn216076] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:14:22.934Z I CONTROL [conn216076] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:14:42.925Z I - [conn216093] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:14:42.935Z I CONTROL [conn216093] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:15:02.942Z I - [conn183343] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:15:02.951Z I CONTROL [conn183343] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:15:22.923Z I - [conn179981] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:15:22.933Z I CONTROL [conn179981] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:15:42.911Z I - [conn205930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:15:42.920Z I CONTROL [conn205930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:16:02.922Z I - [conn205487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:16:02.932Z I CONTROL [conn205487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:16:22.918Z I - [conn170812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:16:22.928Z I CONTROL [conn170812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:16:42.925Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:16:42.934Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:17:02.929Z I - [conn205787] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:17:02.939Z I CONTROL [conn205787] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:17:22.921Z I - [conn205515] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:17:22.931Z I CONTROL [conn205515] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:17:42.925Z I - [conn215915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:17:42.934Z I CONTROL [conn215915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:18:02.933Z I - [conn216060] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:18:02.943Z I CONTROL [conn216060] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:18:22.921Z I - [conn205589] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:18:22.931Z I CONTROL [conn205589] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:18:42.931Z I - [conn216134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:18:42.941Z I CONTROL [conn216134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:19:02.932Z I - [conn215148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:19:02.942Z I CONTROL [conn215148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:19:22.917Z I - [conn215923] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:19:22.927Z I CONTROL [conn215923] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:19:42.921Z I - [conn205822] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:19:42.931Z I CONTROL [conn205822] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:20:02.924Z I - [conn216110] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:20:02.934Z I CONTROL [conn216110] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:20:22.918Z I - [conn206332] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:20:22.928Z I CONTROL [conn206332] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:20:42.926Z I - [conn216096] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:20:42.936Z I CONTROL [conn216096] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:21:02.924Z I - [conn205609] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:21:02.933Z I CONTROL [conn205609] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:21:22.911Z I - [conn205940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:21:22.921Z I CONTROL [conn205940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:21:26.265Z I - [conn216066] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:21:26.275Z I CONTROL [conn216066] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:21:42.917Z I - [conn205935] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:21:42.927Z I CONTROL [conn205935] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:22:02.907Z I - [conn216137] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:22:02.916Z I CONTROL [conn216137] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:22:22.908Z I - [conn205598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:22:22.918Z I CONTROL [conn205598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:22:42.919Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:22:42.928Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:23:02.939Z I - [conn215910] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:23:02.951Z I CONTROL [conn215910] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:23:22.909Z I - [conn206314] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:23:22.919Z I CONTROL [conn206314] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:23:42.920Z I - [conn205818] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:23:42.931Z I CONTROL [conn205818] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:24:02.920Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:24:02.929Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:24:22.918Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:24:22.928Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:24:42.927Z I - [conn205943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:24:42.937Z I CONTROL [conn205943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:25:02.913Z I - [conn205818] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:25:02.922Z I CONTROL [conn205818] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:25:22.907Z I - [conn205787] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:25:22.917Z I CONTROL [conn205787] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:25:42.915Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:25:42.925Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:26:02.910Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:26:02.919Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:26:22.906Z I - [conn205930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:26:22.916Z I CONTROL [conn205930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:26:42.909Z I - [conn205499] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:26:42.919Z I CONTROL [conn205499] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:27:02.915Z I - [conn206314] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:27:02.925Z I CONTROL [conn206314] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:27:22.906Z I - [conn205948] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:27:22.916Z I CONTROL [conn205948] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:27:42.922Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:27:42.931Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:28:02.928Z I - [conn215148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:28:02.938Z I CONTROL [conn215148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:28:22.905Z I - [conn205826] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:28:22.915Z I CONTROL [conn205826] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:28:42.938Z I - [conn205658] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:28:42.947Z I CONTROL [conn205658] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:29:02.916Z I - [conn206771] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:29:02.925Z I CONTROL [conn206771] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:29:22.916Z I - [conn206298] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:29:22.926Z I CONTROL [conn206298] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:29:42.914Z I - [conn205628] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:29:42.923Z I CONTROL [conn205628] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:30:02.933Z I - [conn205826] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:30:02.942Z I CONTROL [conn205826] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:30:22.911Z I - [conn215854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:30:22.920Z I CONTROL [conn215854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:30:42.915Z I - [conn205515] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:30:42.925Z I CONTROL [conn205515] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:31:02.914Z I - [conn205561] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:31:02.923Z I CONTROL [conn205561] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:31:22.902Z I - [conn206256] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:31:22.911Z I CONTROL [conn206256] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:31:42.903Z I - [conn205683] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:31:42.913Z I CONTROL [conn205683] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:32:02.916Z I - [conn205573] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:32:02.925Z I CONTROL [conn205573] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:32:22.913Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:32:22.922Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:32:42.920Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:32:42.929Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:33:02.901Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:33:02.911Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:33:22.914Z I - [conn205816] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:33:22.924Z I CONTROL [conn205816] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:33:42.917Z I - [conn206810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:33:42.927Z I CONTROL [conn206810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:34:02.902Z I - [conn206810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:34:02.911Z I CONTROL [conn206810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:34:22.900Z I - [conn215914] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:34:22.909Z I CONTROL [conn215914] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:34:42.923Z I - [conn216066] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:34:42.932Z I CONTROL [conn216066] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:35:02.909Z I - [conn215976] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:35:02.919Z I CONTROL [conn215976] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:35:22.913Z I - [conn205858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:35:22.923Z I CONTROL [conn205858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:35:42.908Z I - [conn183343] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:35:42.918Z I CONTROL [conn183343] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:36:02.903Z I - [conn206773] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:36:02.912Z I CONTROL [conn206773] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:36:12.628Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:36:12.638Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:36:22.911Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:36:22.921Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:36:42.911Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:36:42.921Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:37:02.900Z I - [conn216093] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:37:02.910Z I CONTROL [conn216093] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:37:22.911Z I - [conn183095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:37:22.920Z I CONTROL [conn183095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:37:42.905Z I - [conn205554] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:37:42.915Z I CONTROL [conn205554] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:38:02.925Z I - [conn206810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:38:02.935Z I CONTROL [conn206810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:38:22.908Z I - [conn215851] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:38:22.917Z I CONTROL [conn215851] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:38:42.900Z I - [conn205562] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:38:42.910Z I CONTROL [conn205562] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:39:02.911Z I - [conn212599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:39:02.921Z I CONTROL [conn212599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:39:22.927Z I - [conn215923] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:39:22.937Z I CONTROL [conn215923] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:39:42.915Z I - [conn212228] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:39:42.925Z I CONTROL [conn212228] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:40:02.914Z I - [conn205515] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:40:02.924Z I CONTROL [conn205515] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:40:22.899Z I - [conn205776] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:40:22.908Z I CONTROL [conn205776] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:40:42.908Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:40:42.917Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:41:02.896Z I - [conn215944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:41:02.906Z I CONTROL [conn215944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:41:22.916Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:41:22.926Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:41:42.917Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:41:42.927Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:42:02.895Z I - [conn216066] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:42:02.905Z I CONTROL [conn216066] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:42:22.905Z I - [conn205573] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:42:22.915Z I CONTROL [conn205573] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:42:42.907Z I - [conn205927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:42:42.917Z I CONTROL [conn205927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:43:02.906Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:43:02.916Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:43:22.908Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:43:22.917Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:43:42.905Z I - [conn205646] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:43:42.915Z I CONTROL [conn205646] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:44:02.902Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:44:02.911Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:44:22.906Z I - [conn205942] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:44:22.916Z I CONTROL [conn205942] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:44:42.917Z I - [conn205942] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:44:42.927Z I CONTROL [conn205942] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:45:02.904Z I - [conn215140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:45:02.914Z I CONTROL [conn215140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:45:22.917Z I - [conn216141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:45:22.927Z I CONTROL [conn216141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:45:42.906Z I - [conn205598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:45:42.916Z I CONTROL [conn205598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:46:02.898Z I - [conn205562] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:46:02.908Z I CONTROL [conn205562] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:46:22.916Z I - [conn205473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:46:22.925Z I CONTROL [conn205473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:46:42.897Z I - [conn205486] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:46:42.906Z I CONTROL [conn205486] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:47:02.906Z I - [conn205473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:47:02.915Z I CONTROL [conn205473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:47:22.906Z I - [conn216122] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:47:22.916Z I CONTROL [conn216122] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:53:41.592Z I - [conn215852] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:53:41.601Z I CONTROL [conn215852] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:54:01.522Z I - [conn215958] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:54:01.532Z I CONTROL [conn215958] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:54:21.499Z I - [conn205455] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:54:21.509Z I CONTROL [conn205455] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:54:41.476Z I - [conn215962] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:54:41.486Z I CONTROL [conn215962] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:55:01.469Z I - [conn177467] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:55:01.479Z I CONTROL [conn177467] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:55:21.491Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:55:21.500Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:55:41.478Z I - [conn205871] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:55:41.488Z I CONTROL [conn205871] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:56:01.476Z I - [conn205473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:56:01.486Z I CONTROL [conn205473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:56:21.492Z I - [conn205942] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:56:21.502Z I CONTROL [conn205942] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:56:41.474Z I - [conn205577] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:56:41.483Z I CONTROL [conn205577] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:57:01.488Z I - [conn205942] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:57:01.497Z I CONTROL [conn205942] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:57:21.476Z I - [conn216078] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:57:21.485Z I CONTROL [conn216078] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:57:41.477Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:57:41.486Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:58:01.482Z I - [conn215858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:58:01.491Z I CONTROL [conn215858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:58:21.463Z I - [conn215858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:58:21.473Z I CONTROL [conn215858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:58:41.481Z I - [conn205824] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:58:41.490Z I CONTROL [conn205824] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:59:01.465Z I - [conn205455] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:59:01.474Z I CONTROL [conn205455] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:59:21.475Z I - [conn215913] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:59:21.485Z I CONTROL [conn215913] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:59:41.481Z I - [conn205940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T19:59:41.491Z I CONTROL [conn205940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T19:59:48.795Z I INDEX [conn215535] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T20:00:01.485Z I - [conn205915] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:00:01.495Z I CONTROL [conn205915] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:00:21.475Z I - [conn215941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:00:21.484Z I CONTROL [conn215941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:00:41.464Z I - [conn212596] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:00:41.476Z I CONTROL [conn212596] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:01:01.476Z I - [conn205908] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:01:01.485Z I CONTROL [conn205908] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:01:21.475Z I - [conn179981] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:01:21.485Z I CONTROL [conn179981] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:01:41.481Z I - [conn216095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:01:41.491Z I CONTROL [conn216095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:02:01.475Z I - [conn205822] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:02:01.485Z I CONTROL [conn205822] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:02:21.487Z I - [conn215149] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:02:21.497Z I CONTROL [conn215149] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:02:41.470Z I - [conn216113] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:02:41.484Z I CONTROL [conn216113] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:03:01.478Z I - [conn183171] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:03:01.490Z I CONTROL [conn183171] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:03:21.474Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:03:21.483Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:03:41.466Z I - [conn205881] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:03:41.475Z I CONTROL [conn205881] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:04:01.474Z I - [conn216152] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:04:01.483Z I CONTROL [conn216152] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:04:21.470Z I - [conn216155] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:04:21.481Z I CONTROL [conn216155] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:04:41.475Z I - [conn206773] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:04:41.486Z I CONTROL [conn206773] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:05:01.474Z I - [conn177467] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:05:01.483Z I CONTROL [conn177467] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:05:21.462Z I - [conn170504] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:05:21.472Z I CONTROL [conn170504] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:05:41.464Z I - [conn170504] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:05:41.474Z I CONTROL [conn170504] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:06:01.484Z I - [conn205858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:06:01.494Z I CONTROL [conn205858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:06:21.484Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:06:21.494Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:06:41.486Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:06:41.496Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:07:01.458Z I - [conn215858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:07:01.468Z I CONTROL [conn215858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:07:21.474Z I - [conn183343] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:07:21.483Z I CONTROL [conn183343] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:07:41.473Z I - [conn206261] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:07:41.483Z I CONTROL [conn206261] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:08:01.473Z I - [conn177467] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:08:01.483Z I CONTROL [conn177467] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:08:21.472Z I - [conn205486] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:08:21.481Z I CONTROL [conn205486] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:08:41.466Z I - [conn216053] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:08:41.477Z I CONTROL [conn216053] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:09:01.469Z I - [conn215914] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:09:01.484Z I CONTROL [conn215914] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:09:21.471Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:09:21.480Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:09:41.471Z I - [conn216111] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:09:41.481Z I CONTROL [conn216111] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:10:01.458Z I - [conn215923] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:10:01.467Z I CONTROL [conn215923] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:10:21.466Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:10:21.476Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:10:41.469Z I - [conn216137] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:10:41.479Z I CONTROL [conn216137] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:15:01.001Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:15:01.012Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:15:20.971Z I - [conn216135] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:15:20.981Z I CONTROL [conn216135] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:15:40.985Z I - [conn205598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:15:40.995Z I CONTROL [conn205598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:16:00.982Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:16:00.991Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:16:20.993Z I - [conn215983] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:16:21.003Z I CONTROL [conn215983] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:16:40.983Z I - [conn216083] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:16:40.992Z I CONTROL [conn216083] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:17:00.975Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:17:00.985Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:17:20.988Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:17:20.998Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:17:40.984Z I - [conn205740] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:17:40.993Z I CONTROL [conn205740] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:18:00.996Z I - [conn205630] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:18:01.005Z I CONTROL [conn205630] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:18:20.988Z I - [conn205630] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:18:20.997Z I CONTROL [conn205630] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:18:40.995Z I - [conn216096] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:18:41.004Z I CONTROL [conn216096] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:19:00.990Z I - [conn205598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:19:01.000Z I CONTROL [conn205598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:19:20.989Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:19:20.999Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:19:40.991Z I - [conn205633] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:19:41.001Z I CONTROL [conn205633] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:20:00.989Z I - [conn216134] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:20:01.000Z I CONTROL [conn216134] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:20:20.988Z I - [conn206810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:20:20.997Z I CONTROL [conn206810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:20:40.991Z I - [conn205941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:20:41.000Z I CONTROL [conn205941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:21:00.977Z I - [conn205740] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:21:00.987Z I CONTROL [conn205740] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:21:20.997Z I - [conn205866] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:21:21.007Z I CONTROL [conn205866] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:21:40.988Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:21:40.998Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:22:00.988Z I - [conn205566] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:22:00.998Z I CONTROL [conn205566] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:22:20.995Z I - [conn206255] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:22:21.005Z I CONTROL [conn206255] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:22:40.986Z I - [conn205561] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:22:40.996Z I CONTROL [conn205561] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:23:00.990Z I - [conn205561] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:23:01.000Z I CONTROL [conn205561] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:23:20.977Z I - [conn216077] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:23:20.987Z I CONTROL [conn216077] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:23:40.985Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:23:40.995Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:24:00.979Z I - [conn216080] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:24:00.992Z I CONTROL [conn216080] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:24:20.988Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:24:20.998Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:24:40.993Z I - [conn206332] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:24:41.003Z I CONTROL [conn206332] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:25:00.977Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:25:00.987Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:25:20.986Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:25:21.000Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:25:40.986Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:25:40.996Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:26:00.992Z I - [conn216076] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:26:01.001Z I CONTROL [conn216076] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:26:20.975Z I - [conn205944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:26:20.984Z I CONTROL [conn205944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:26:40.986Z I - [conn179981] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:26:40.996Z I CONTROL [conn179981] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:27:00.982Z I - [conn215852] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:27:00.991Z I CONTROL [conn215852] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:27:20.981Z I - [conn212228] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:27:20.990Z I CONTROL [conn212228] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:27:40.983Z I - [conn183344] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:27:40.992Z I CONTROL [conn183344] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:28:00.975Z I - [conn205943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:28:00.984Z I CONTROL [conn205943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:28:20.976Z I - [conn206317] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:28:20.985Z I CONTROL [conn206317] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:28:40.974Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:28:40.985Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:29:00.978Z I - [conn177467] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:29:00.988Z I CONTROL [conn177467] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:29:20.981Z I - [conn215858] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:29:20.991Z I CONTROL [conn215858] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:29:40.985Z I - [conn205577] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:29:41.000Z I CONTROL [conn205577] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:30:00.985Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:30:00.996Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:30:20.989Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:30:20.999Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:30:40.985Z I - [conn205828] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:30:40.994Z I CONTROL [conn205828] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:31:00.982Z I - [conn206318] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:31:00.992Z I CONTROL [conn206318] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:31:20.981Z I - [conn183171] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:31:20.990Z I CONTROL [conn183171] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:31:40.984Z I - [conn205928] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:31:40.993Z I CONTROL [conn205928] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:32:00.982Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:32:00.991Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:32:20.979Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:32:20.989Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:32:40.972Z I - [conn206332] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:32:40.981Z I CONTROL [conn206332] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:33:00.980Z I - [conn215960] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:33:00.990Z I CONTROL [conn215960] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:33:20.985Z I - [conn212598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:33:20.994Z I CONTROL [conn212598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:33:40.971Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:33:40.980Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:34:00.970Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:34:00.979Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:34:20.981Z I - [conn170812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:34:20.991Z I CONTROL [conn170812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:34:40.968Z I - [conn205499] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:34:40.978Z I CONTROL [conn205499] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:35:00.984Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:35:00.994Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:35:20.970Z I - [conn205675] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:35:20.979Z I CONTROL [conn205675] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:35:40.989Z I - [conn207417] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:35:40.998Z I CONTROL [conn207417] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:36:00.968Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:36:00.979Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:36:20.969Z I - [conn205828] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:36:20.979Z I CONTROL [conn205828] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:36:40.982Z I - [conn205499] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:36:40.992Z I CONTROL [conn205499] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:37:00.967Z I - [conn215956] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:37:00.977Z I CONTROL [conn215956] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:37:20.973Z I - [conn205554] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:37:20.984Z I CONTROL [conn205554] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:37:40.979Z I - [conn215917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:37:40.990Z I CONTROL [conn215917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:38:00.970Z I - [conn205791] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:38:00.980Z I CONTROL [conn205791] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:38:21.007Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:38:21.017Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:38:40.967Z I - [conn216137] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:38:40.976Z I CONTROL [conn216137] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:39:00.978Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:39:00.988Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:39:20.990Z I - [conn206314] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:39:21.000Z I CONTROL [conn206314] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:39:40.971Z I - [conn205599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:39:40.981Z I CONTROL [conn205599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:40:00.979Z I - [conn215908] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:40:00.988Z I CONTROL [conn215908] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:40:20.975Z I - [conn215960] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:40:20.985Z I CONTROL [conn215960] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:40:40.986Z I - [conn216104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:40:40.996Z I CONTROL [conn216104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:41:00.980Z I - [conn215140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:41:00.989Z I CONTROL [conn215140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:41:20.980Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:41:20.990Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:41:40.974Z I - [conn215960] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:41:40.983Z I CONTROL [conn215960] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:42:00.977Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:42:00.987Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:42:20.976Z I - [conn206256] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:42:20.988Z I CONTROL [conn206256] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:42:40.979Z I - [conn215140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:42:40.995Z I CONTROL [conn215140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:43:00.987Z I - [conn205569] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:43:00.996Z I CONTROL [conn205569] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:43:20.977Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:43:20.986Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:43:40.972Z I - [conn216111] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:43:40.982Z I CONTROL [conn216111] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:44:00.989Z I - [conn205903] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:44:01.000Z I CONTROL [conn205903] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:44:20.981Z I - [conn183095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:44:20.991Z I CONTROL [conn183095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:44:40.976Z I - [conn206298] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:44:40.986Z I CONTROL [conn206298] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:45:00.977Z I - [conn215946] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:45:00.986Z I CONTROL [conn215946] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:45:20.981Z I - [conn205599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:45:20.991Z I CONTROL [conn205599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:45:40.978Z I - [conn206298] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:45:40.987Z I CONTROL [conn206298] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:46:00.979Z I - [conn206298] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:46:00.990Z I CONTROL [conn206298] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:46:20.984Z I - [conn216139] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:46:20.994Z I CONTROL [conn216139] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:46:40.965Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:46:40.975Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:47:00.980Z I - [conn205940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:47:00.989Z I CONTROL [conn205940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:47:20.975Z I - [conn183103] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:47:20.986Z I CONTROL [conn183103] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:47:40.976Z I - [conn216146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:47:40.986Z I CONTROL [conn216146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:48:00.971Z I - [conn215971] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:48:00.982Z I CONTROL [conn215971] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:48:20.978Z I - [conn205943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:48:20.987Z I CONTROL [conn205943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:48:40.973Z I - [conn206799] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:48:40.983Z I CONTROL [conn206799] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:49:00.971Z I - [conn215941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:49:00.984Z I CONTROL [conn215941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:49:20.974Z I - [conn216139] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:49:20.988Z I CONTROL [conn216139] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:49:40.977Z I - [conn205566] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:49:40.987Z I CONTROL [conn205566] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:50:00.975Z I - [conn206809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:50:00.987Z I CONTROL [conn206809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:50:20.973Z I - [conn205625] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:50:20.983Z I CONTROL [conn205625] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:50:40.974Z I - [conn215930] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:50:40.984Z I CONTROL [conn215930] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:51:00.973Z I - [conn206316] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:51:00.983Z I CONTROL [conn206316] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:51:20.999Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:51:21.009Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:51:40.975Z I - [conn216116] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:51:40.986Z I CONTROL [conn216116] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:52:00.986Z I - [conn205631] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:52:00.997Z I CONTROL [conn205631] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:52:20.984Z I - [conn205745] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:52:20.994Z I CONTROL [conn205745] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:52:40.974Z I - [conn216096] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:52:40.987Z I CONTROL [conn216096] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:53:00.974Z I - [conn170504] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:53:00.984Z I CONTROL [conn170504] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:53:20.968Z I - [conn170473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:53:20.994Z I CONTROL [conn170473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:53:40.973Z I - [conn205486] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:53:40.983Z I CONTROL [conn205486] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:54:00.990Z I - [conn206773] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:54:01.001Z I CONTROL [conn206773] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:54:20.962Z I - [conn215952] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:54:20.972Z I CONTROL [conn215952] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:54:40.974Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:54:40.984Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:55:00.984Z I - [conn216093] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:55:00.994Z I CONTROL [conn216093] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:55:20.965Z I - [conn215941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:55:20.975Z I CONTROL [conn215941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:55:40.972Z I - [conn205515] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:55:40.981Z I CONTROL [conn205515] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:56:00.963Z I - [conn205900] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:56:00.974Z I CONTROL [conn205900] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:56:20.973Z I - [conn205752] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:56:20.985Z I CONTROL [conn205752] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:56:40.979Z I - [conn215916] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:56:40.989Z I CONTROL [conn215916] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:57:00.973Z I - [conn216161] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:57:00.982Z I CONTROL [conn216161] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:57:20.969Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:57:20.978Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:57:40.963Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:57:40.973Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:58:00.961Z I - [conn205856] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:58:00.972Z I CONTROL [conn205856] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:58:20.974Z I - [conn216083] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:58:20.984Z I CONTROL [conn216083] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:58:40.971Z I - [conn215907] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:58:40.981Z I CONTROL [conn215907] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:59:00.968Z I - [conn216148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:59:00.977Z I CONTROL [conn216148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:59:20.961Z I - [conn216129] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:59:20.971Z I CONTROL [conn216129] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:59:40.971Z I - [conn206810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T20:59:40.980Z I CONTROL [conn206810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T20:59:48.619Z I INDEX [conn215568] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T21:00:00.973Z I - [conn215138] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:00:00.983Z I CONTROL [conn215138] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:00:20.968Z I - [conn177378] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:00:20.978Z I CONTROL [conn177378] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:00:40.971Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:00:40.981Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:01:00.972Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:01:00.982Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:01:20.957Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:01:20.966Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:01:37.895Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:01:37.905Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:01:40.970Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:01:40.979Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:02:00.981Z I - [conn205736] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:02:00.991Z I CONTROL [conn205736] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:02:20.968Z I - [conn215914] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:02:20.977Z I CONTROL [conn215914] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:02:40.976Z I - [conn216149] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:02:40.986Z I CONTROL [conn216149] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:03:00.965Z I - [conn206317] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:03:00.975Z I CONTROL [conn206317] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:03:20.967Z I - [conn215944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:03:20.977Z I CONTROL [conn215944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:03:40.978Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:03:40.988Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:04:00.971Z I - [conn205633] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:04:00.980Z I CONTROL [conn205633] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:04:20.981Z I - [conn205518] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:04:20.990Z I CONTROL [conn205518] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:04:40.965Z I - [conn206794] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:04:40.975Z I CONTROL [conn206794] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:05:00.965Z I - [conn205877] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:05:00.975Z I CONTROL [conn205877] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:05:20.969Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:05:20.979Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:05:40.970Z I - [conn205487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:05:40.980Z I CONTROL [conn205487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:06:00.968Z I - [conn205904] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:06:00.977Z I CONTROL [conn205904] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:06:21.313Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:06:21.322Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:06:40.970Z I - [conn205577] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:06:40.979Z I CONTROL [conn205577] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:07:00.970Z I - [conn206799] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:07:00.979Z I CONTROL [conn206799] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:07:20.966Z I - [conn216144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:07:20.976Z I CONTROL [conn216144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:07:40.964Z I - [conn215852] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:07:40.973Z I CONTROL [conn215852] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:08:00.965Z I - [conn206772] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:08:00.975Z I CONTROL [conn206772] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:08:20.969Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:08:20.979Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:08:40.976Z I - [conn205928] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:08:40.986Z I CONTROL [conn205928] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:09:00.969Z I - [conn205451] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:09:00.979Z I CONTROL [conn205451] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:09:20.962Z I - [conn215908] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:09:20.972Z I CONTROL [conn215908] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:09:40.973Z I - [conn207411] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:09:40.983Z I CONTROL [conn207411] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:10:00.967Z I - [conn216060] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:10:00.977Z I CONTROL [conn216060] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:10:20.968Z I - [conn205795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:10:20.977Z I CONTROL [conn205795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:10:40.974Z I - [conn205814] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:10:40.983Z I CONTROL [conn205814] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:11:00.964Z I - [conn205804] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:11:00.974Z I CONTROL [conn205804] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:11:20.965Z I - [conn205931] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:11:20.975Z I CONTROL [conn205931] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:11:40.963Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:11:40.973Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:12:00.968Z I - [conn205816] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:12:00.977Z I CONTROL [conn205816] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:12:20.967Z I - [conn206775] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:12:20.977Z I CONTROL [conn206775] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:12:40.966Z I - [conn216148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:12:40.975Z I CONTROL [conn216148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:13:00.965Z I - [conn216148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:13:00.974Z I CONTROL [conn216148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:13:20.966Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:13:20.975Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:13:40.964Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:13:40.973Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:14:00.961Z I - [conn216140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:14:00.971Z I CONTROL [conn216140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:14:20.964Z I - [conn205478] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:14:20.973Z I CONTROL [conn205478] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:14:40.974Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:14:40.983Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:15:00.978Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:15:00.987Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:15:11.111Z I - [conn205554] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:15:11.121Z I CONTROL [conn205554] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:15:20.960Z I - [conn215976] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:15:20.970Z I CONTROL [conn215976] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:15:40.959Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:15:40.968Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:16:00.974Z I - [conn215973] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:16:00.984Z I CONTROL [conn215973] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:16:20.983Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:16:20.992Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:16:40.963Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:16:40.972Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:17:00.980Z I - [conn205561] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:17:00.990Z I CONTROL [conn205561] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:17:20.979Z I - [conn216155] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:17:20.989Z I CONTROL [conn216155] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:17:40.973Z I - [conn216104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:17:40.983Z I CONTROL [conn216104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:18:00.963Z I - [conn215850] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:18:00.973Z I CONTROL [conn215850] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:18:20.959Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:18:20.968Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:18:40.963Z I - [conn216083] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:18:40.973Z I CONTROL [conn216083] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:19:00.970Z I - [conn205937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:19:00.980Z I CONTROL [conn205937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:19:20.962Z I - [conn216155] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:19:20.972Z I CONTROL [conn216155] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:19:40.969Z I - [conn183342] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:19:40.979Z I CONTROL [conn183342] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:20:00.965Z I - [conn183104] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:20:00.975Z I CONTROL [conn183104] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:20:20.964Z I - [conn205571] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:20:20.974Z I CONTROL [conn205571] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:20:40.951Z I - [conn171685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:20:40.961Z I CONTROL [conn171685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:21:00.961Z I - [conn205482] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:21:00.970Z I CONTROL [conn205482] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:21:20.959Z I - [conn205819] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:21:20.969Z I CONTROL [conn205819] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:21:40.967Z I - [conn205804] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:21:40.976Z I CONTROL [conn205804] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:22:00.959Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:22:00.968Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:22:20.980Z I - [conn216161] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:22:20.990Z I CONTROL [conn216161] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:22:40.957Z I - [conn206255] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:22:40.967Z I CONTROL [conn206255] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:23:00.958Z I - [conn171685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:23:00.967Z I CONTROL [conn171685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:23:20.966Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:23:20.975Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:23:40.961Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:23:40.971Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:24:00.956Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:24:00.966Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:24:20.955Z I - [conn216141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:24:20.965Z I CONTROL [conn216141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:24:40.958Z I - [conn205478] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:24:40.967Z I CONTROL [conn205478] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:25:00.959Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:25:00.968Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:25:20.967Z I - [conn205573] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:25:20.977Z I CONTROL [conn205573] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:25:40.969Z I - [conn205573] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:25:40.978Z I CONTROL [conn205573] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:26:00.970Z I - [conn205691] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:26:00.980Z I CONTROL [conn205691] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:26:20.957Z I - [conn171685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:26:20.966Z I CONTROL [conn171685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:26:40.965Z I - [conn215914] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:26:40.975Z I CONTROL [conn215914] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:27:00.960Z I - [conn215852] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:27:00.969Z I CONTROL [conn215852] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:27:20.970Z I - [conn171685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:27:20.980Z I CONTROL [conn171685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:27:40.956Z I - [conn206317] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:27:40.966Z I CONTROL [conn206317] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:28:00.963Z I - [conn215976] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:28:00.973Z I CONTROL [conn215976] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:28:20.963Z I - [conn177378] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:28:20.972Z I CONTROL [conn177378] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:28:40.946Z I - [conn215976] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:28:40.955Z I CONTROL [conn215976] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:29:00.967Z I - [conn205826] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:29:00.977Z I CONTROL [conn205826] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:29:20.957Z I - [conn205587] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:29:20.967Z I CONTROL [conn205587] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:29:40.957Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:29:40.966Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:30:00.965Z I - [conn216083] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:30:00.975Z I CONTROL [conn216083] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:30:20.955Z I - [conn177419] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:30:20.965Z I CONTROL [conn177419] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:30:40.954Z I - [conn216064] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:30:40.964Z I CONTROL [conn216064] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:31:00.958Z I - [conn205776] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:31:00.967Z I CONTROL [conn205776] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:31:20.942Z I - [conn215946] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:31:20.952Z I CONTROL [conn215946] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:31:40.963Z I - [conn205917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:31:40.973Z I CONTROL [conn205917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:32:00.958Z I - [conn205589] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:32:00.968Z I CONTROL [conn205589] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:32:20.955Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:32:20.964Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:32:40.966Z I - [conn206799] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:32:40.976Z I CONTROL [conn206799] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:33:00.955Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:33:00.965Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:33:20.955Z I - [conn205826] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:33:20.964Z I CONTROL [conn205826] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:33:40.942Z I - [conn215943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:33:40.951Z I CONTROL [conn215943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:34:00.957Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:34:00.966Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:34:20.961Z I - [conn205839] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:34:20.971Z I CONTROL [conn205839] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:34:40.964Z I - [conn212228] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:34:40.974Z I CONTROL [conn212228] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:35:00.956Z I - [conn205628] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:35:00.966Z I CONTROL [conn205628] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:35:20.954Z I - [conn179981] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:35:20.963Z I CONTROL [conn179981] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:35:40.950Z I - [conn216155] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:35:40.960Z I CONTROL [conn216155] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:36:00.953Z I - [conn206315] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:36:00.963Z I CONTROL [conn206315] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:36:20.938Z I - [conn216093] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:36:20.948Z I CONTROL [conn216093] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:36:40.952Z I - [conn205478] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:36:40.961Z I CONTROL [conn205478] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:37:00.962Z I - [conn205499] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:37:00.972Z I CONTROL [conn205499] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:37:20.957Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:37:20.967Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:37:40.953Z I - [conn205736] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:37:40.962Z I CONTROL [conn205736] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:38:00.954Z I - [conn205454] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:38:00.964Z I CONTROL [conn205454] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:38:20.955Z I - [conn205464] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:38:20.965Z I CONTROL [conn205464] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:38:40.959Z I - [conn205787] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:38:40.969Z I CONTROL [conn205787] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:39:00.951Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:39:00.961Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:39:20.949Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:39:20.959Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:39:40.952Z I - [conn206259] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:39:40.962Z I CONTROL [conn206259] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:40:00.958Z I - [conn205589] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:40:00.968Z I CONTROL [conn205589] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:40:20.953Z I - [conn215962] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:40:20.962Z I CONTROL [conn215962] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:40:40.951Z I - [conn216083] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:40:40.961Z I CONTROL [conn216083] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:41:00.953Z I - [conn205630] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:41:00.963Z I CONTROL [conn205630] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:41:20.956Z I - [conn205630] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:41:20.965Z I CONTROL [conn205630] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:41:40.949Z I - [conn205484] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:41:40.958Z I CONTROL [conn205484] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:42:00.952Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:42:00.961Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:42:20.962Z I - [conn216139] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:42:20.972Z I CONTROL [conn216139] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:42:40.959Z I - [conn205877] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:42:40.969Z I CONTROL [conn205877] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:43:00.956Z I - [conn205853] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:43:00.966Z I CONTROL [conn205853] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:43:20.961Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:43:20.970Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:43:40.962Z I - [conn177807] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:43:40.973Z I CONTROL [conn177807] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:44:00.950Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:44:00.960Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:44:20.951Z I - [conn205685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:44:20.960Z I CONTROL [conn205685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:44:40.953Z I - [conn212225] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:44:40.962Z I CONTROL [conn212225] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:45:00.952Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:45:00.962Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:45:20.958Z I - [conn216078] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:45:20.968Z I CONTROL [conn216078] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:45:40.956Z I - [conn215922] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:45:40.966Z I CONTROL [conn215922] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:46:00.949Z I - [conn215852] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:46:00.959Z I CONTROL [conn215852] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:46:20.949Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:46:20.958Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:46:40.947Z I - [conn215913] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:46:40.957Z I CONTROL [conn215913] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:47:00.956Z I - [conn215149] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:47:00.966Z I CONTROL [conn215149] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:47:20.951Z I - [conn205554] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:47:20.960Z I CONTROL [conn205554] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:47:40.948Z I - [conn205943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:47:40.958Z I CONTROL [conn205943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:48:00.955Z I - [conn205859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:48:00.965Z I CONTROL [conn205859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:48:20.947Z I - [conn206330] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:48:20.956Z I CONTROL [conn206330] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:48:40.957Z I - [conn205818] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:48:40.967Z I CONTROL [conn205818] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:49:00.944Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:49:00.953Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:49:20.947Z I - [conn216137] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:49:20.957Z I CONTROL [conn216137] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:49:40.946Z I - [conn170473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:49:40.956Z I CONTROL [conn170473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:50:00.950Z I - [conn212226] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:50:00.959Z I CONTROL [conn212226] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:50:20.947Z I - [conn205822] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:50:20.956Z I CONTROL [conn205822] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:50:40.957Z I - [conn216116] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:50:40.967Z I CONTROL [conn216116] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:51:00.945Z I - [conn205816] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:51:00.955Z I CONTROL [conn205816] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:51:20.947Z I - [conn205940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:51:20.957Z I CONTROL [conn205940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:51:40.951Z I - [conn205569] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:51:40.961Z I CONTROL [conn205569] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:52:00.948Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T21:52:00.958Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T21:59:48.609Z I INDEX [conn215568] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T22:59:48.534Z I INDEX [conn215568] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-28T23:00:05.973Z I - [conn205482] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:00:05.982Z I CONTROL [conn205482] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:00:25.975Z I - [conn177807] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:00:25.985Z I CONTROL [conn177807] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:00:45.968Z I - [conn206330] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:00:45.978Z I CONTROL [conn206330] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:01:05.968Z I - [conn205804] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:01:05.978Z I CONTROL [conn205804] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:01:25.974Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:01:25.984Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:01:45.968Z I - [conn205675] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:01:45.977Z I CONTROL [conn205675] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:02:05.969Z I - [conn216060] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:02:05.979Z I CONTROL [conn216060] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:02:25.973Z I - [conn215877] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:02:25.983Z I CONTROL [conn215877] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:02:45.975Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:02:45.985Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:03:05.971Z I - [conn205685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:03:05.981Z I CONTROL [conn205685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:03:25.975Z I - [conn205478] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:03:25.984Z I CONTROL [conn205478] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:03:45.978Z I - [conn205940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:03:45.988Z I CONTROL [conn205940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:04:05.974Z I - [conn216065] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:04:05.984Z I CONTROL [conn216065] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:04:25.974Z I - [conn205482] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:04:25.984Z I CONTROL [conn205482] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:04:45.964Z I - [conn215923] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:04:45.974Z I CONTROL [conn215923] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:05:05.967Z I - [conn216149] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:05:05.977Z I CONTROL [conn216149] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:05:25.964Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:05:25.974Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:05:45.974Z I - [conn183095] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:05:45.983Z I CONTROL [conn183095] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:06:05.975Z I - [conn183114] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:06:05.984Z I CONTROL [conn183114] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:06:25.977Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:06:25.987Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:06:45.979Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:06:45.989Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:07:05.965Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:07:05.975Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:07:25.978Z I - [conn205917] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:07:25.988Z I CONTROL [conn205917] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:07:45.974Z I - [conn205482] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:07:45.983Z I CONTROL [conn205482] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:08:05.985Z I - [conn216101] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:08:05.995Z I CONTROL [conn216101] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:08:25.964Z I - [conn183171] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:08:25.974Z I CONTROL [conn183171] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:08:45.964Z I - [conn205598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:08:45.973Z I CONTROL [conn205598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:09:05.979Z I - [conn207411] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:09:05.989Z I CONTROL [conn207411] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:09:25.964Z I - [conn205927] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:09:25.974Z I CONTROL [conn205927] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:09:45.964Z I - [conn205562] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:09:45.973Z I CONTROL [conn205562] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:10:05.967Z I - [conn205940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:10:05.976Z I CONTROL [conn205940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:10:25.983Z I - [conn177807] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:10:25.993Z I CONTROL [conn177807] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:10:45.977Z I - [conn205562] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:10:45.986Z I CONTROL [conn205562] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:11:05.967Z I - [conn205824] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:11:05.976Z I CONTROL [conn205824] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:11:25.989Z I - [conn216149] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:11:25.998Z I CONTROL [conn216149] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:11:45.964Z I - [conn205824] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:11:45.973Z I CONTROL [conn205824] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:12:05.962Z I - [conn206330] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:12:05.972Z I CONTROL [conn206330] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:12:25.966Z I - [conn205740] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:12:25.976Z I CONTROL [conn205740] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:12:45.979Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:12:45.989Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:13:05.965Z I - [conn215944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:13:05.975Z I CONTROL [conn215944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:13:25.962Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:13:25.971Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:13:45.971Z I - [conn205740] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:13:45.980Z I CONTROL [conn205740] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:14:05.958Z I - [conn215944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:14:05.968Z I CONTROL [conn215944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:14:25.971Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:14:25.980Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:14:45.966Z I - [conn216066] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:14:45.976Z I CONTROL [conn216066] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:15:05.968Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:15:05.977Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:15:25.968Z I - [conn215914] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:15:25.978Z I CONTROL [conn215914] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:15:45.977Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:15:45.986Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:16:05.966Z I - [conn216065] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:16:05.976Z I CONTROL [conn216065] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:16:25.964Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:16:25.974Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:16:45.968Z I - [conn206810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:16:45.978Z I CONTROL [conn206810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:17:05.968Z I - [conn205929] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:17:05.977Z I CONTROL [conn205929] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:17:25.960Z I - [conn205929] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:17:25.969Z I CONTROL [conn205929] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:17:45.960Z I - [conn205788] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:17:45.969Z I CONTROL [conn205788] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:18:05.962Z I - [conn205808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:18:05.971Z I CONTROL [conn205808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:18:25.970Z I - [conn205808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:18:25.980Z I CONTROL [conn205808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:18:45.971Z I - [conn216144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:18:45.981Z I CONTROL [conn216144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:19:05.971Z I - [conn205740] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:19:05.980Z I CONTROL [conn205740] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:19:25.968Z I - [conn205598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:19:25.978Z I CONTROL [conn205598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:19:45.968Z I - [conn205573] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:19:45.978Z I CONTROL [conn205573] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:20:05.961Z I - [conn216116] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:20:05.970Z I CONTROL [conn216116] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:20:25.971Z I - [conn212596] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:20:25.981Z I CONTROL [conn212596] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:20:45.981Z I - [conn205871] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:20:45.990Z I CONTROL [conn205871] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:21:05.962Z I - [conn205646] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:21:05.972Z I CONTROL [conn205646] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:21:25.970Z I - [conn183102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:21:25.980Z I CONTROL [conn183102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:21:45.979Z I - [conn216116] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:21:45.988Z I CONTROL [conn216116] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:22:05.965Z I - [conn215875] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:22:05.974Z I CONTROL [conn215875] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:22:25.957Z I - [conn205740] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:22:25.967Z I CONTROL [conn205740] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:22:45.975Z I - [conn212596] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:22:45.985Z I CONTROL [conn212596] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:23:05.971Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:23:05.981Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:23:25.964Z I - [conn205646] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:23:25.973Z I CONTROL [conn205646] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:23:45.962Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:23:45.972Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:24:05.967Z I - [conn215937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:24:05.976Z I CONTROL [conn215937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:24:25.965Z I - [conn205824] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:24:25.975Z I CONTROL [conn205824] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:24:45.959Z I - [conn205740] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:24:45.969Z I CONTROL [conn205740] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:25:05.986Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:25:05.995Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:25:25.967Z I - [conn215944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:25:25.977Z I CONTROL [conn215944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:25:45.956Z I - [conn205737] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:25:45.966Z I CONTROL [conn205737] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:26:05.964Z I - [conn205691] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:26:05.974Z I CONTROL [conn205691] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:26:25.965Z I - [conn215958] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:26:25.974Z I CONTROL [conn215958] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:26:45.968Z I - [conn215958] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:26:45.977Z I CONTROL [conn215958] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:27:05.974Z I - [conn215937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:27:05.984Z I CONTROL [conn215937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:27:25.964Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:27:25.974Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:27:45.967Z I - [conn215919] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:27:45.977Z I CONTROL [conn215919] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:28:05.964Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:28:05.973Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:28:25.966Z I - [conn170473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:28:25.975Z I CONTROL [conn170473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:28:45.961Z I - [conn215877] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:28:45.971Z I CONTROL [conn215877] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:29:05.965Z I - [conn177808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:29:05.975Z I CONTROL [conn177808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:29:25.955Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:29:25.964Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:29:45.963Z I - [conn205756] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:29:45.973Z I CONTROL [conn205756] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:30:05.974Z I - [conn205675] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:30:05.983Z I CONTROL [conn205675] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:30:25.967Z I - [conn205824] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:30:25.977Z I CONTROL [conn205824] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:30:45.954Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:30:45.963Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:31:05.967Z I - [conn206318] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:31:05.977Z I CONTROL [conn206318] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:31:25.973Z I - [conn177419] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:31:25.983Z I CONTROL [conn177419] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:31:45.962Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:31:45.971Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:32:05.966Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:32:05.975Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:32:25.955Z I - [conn205484] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:32:25.964Z I CONTROL [conn205484] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:32:45.959Z I - [conn215926] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:32:45.968Z I CONTROL [conn215926] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:33:05.952Z I - [conn215926] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:33:05.962Z I CONTROL [conn215926] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:33:25.960Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:33:25.969Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:33:45.974Z I - [conn205831] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:33:45.985Z I CONTROL [conn205831] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:34:05.969Z I - [conn170473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:34:05.978Z I CONTROL [conn170473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:34:25.966Z I - [conn206808] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:34:25.975Z I CONTROL [conn206808] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:34:45.964Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:34:45.974Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:35:05.969Z I - [conn215914] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:35:05.978Z I CONTROL [conn215914] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:35:25.965Z I - [conn205803] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:35:25.975Z I CONTROL [conn205803] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:35:45.960Z I - [conn206775] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:35:45.969Z I CONTROL [conn206775] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:36:05.967Z I - [conn205918] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:36:05.977Z I CONTROL [conn205918] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:36:25.967Z I - [conn215980] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:36:25.977Z I CONTROL [conn215980] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:36:45.959Z I - [conn206315] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:36:45.969Z I CONTROL [conn206315] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:37:05.954Z I - [conn205774] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:37:05.964Z I CONTROL [conn205774] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:37:25.984Z I - [conn205745] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:37:25.994Z I CONTROL [conn205745] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:37:45.958Z I - [conn205630] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:37:45.969Z I CONTROL [conn205630] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:38:05.957Z I - [conn216082] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:38:05.967Z I CONTROL [conn216082] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:38:25.955Z I - [conn205601] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:38:25.965Z I CONTROL [conn205601] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:38:45.949Z I - [conn206772] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:38:45.959Z I CONTROL [conn206772] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:39:05.963Z I - [conn205682] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:39:05.973Z I CONTROL [conn205682] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:39:25.959Z I - [conn216099] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:39:25.968Z I CONTROL [conn216099] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:39:45.962Z I - [conn215138] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:39:45.972Z I CONTROL [conn215138] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:40:05.950Z I - [conn205528] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:40:05.959Z I CONTROL [conn205528] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:40:25.959Z I - [conn216128] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:40:25.969Z I CONTROL [conn216128] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:40:45.966Z I - [conn205840] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:40:45.975Z I CONTROL [conn205840] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:41:05.960Z I - [conn205707] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:41:05.970Z I CONTROL [conn205707] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:41:25.949Z I - [conn205707] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:41:25.959Z I CONTROL [conn205707] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:41:45.960Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:41:45.970Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:42:05.958Z I - [conn215876] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:42:05.968Z I CONTROL [conn215876] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:42:25.967Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:42:25.977Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:42:45.969Z I - [conn205798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:42:45.978Z I CONTROL [conn205798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:43:05.966Z I - [conn215138] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:43:05.976Z I CONTROL [conn215138] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:43:25.963Z I - [conn205599] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:43:25.973Z I CONTROL [conn205599] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:43:45.945Z I - [conn216060] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:43:45.956Z I CONTROL [conn216060] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:44:05.953Z I - [conn216060] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:44:05.963Z I CONTROL [conn216060] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:44:25.957Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:44:25.967Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:44:45.954Z I - [conn215138] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:44:45.965Z I CONTROL [conn215138] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:45:05.967Z I - [conn205776] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:45:05.976Z I CONTROL [conn205776] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:45:25.960Z I - [conn205814] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:45:25.969Z I CONTROL [conn205814] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:45:45.946Z I - [conn205456] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:45:45.955Z I CONTROL [conn205456] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:46:05.957Z I - [conn205604] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:46:05.967Z I CONTROL [conn205604] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:46:25.958Z I - [conn207414] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:46:25.968Z I CONTROL [conn207414] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:46:45.945Z I - [conn206258] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:46:45.954Z I CONTROL [conn206258] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:47:05.957Z I - [conn216071] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:47:05.967Z I CONTROL [conn216071] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:47:25.951Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:47:25.962Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:47:45.957Z I - [conn216107] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:47:45.967Z I CONTROL [conn216107] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:48:05.954Z I - [conn215910] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:48:05.963Z I CONTROL [conn215910] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:48:25.944Z I - [conn205815] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:48:25.954Z I CONTROL [conn205815] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:48:45.955Z I - [conn205659] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:48:45.964Z I CONTROL [conn205659] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:49:05.954Z I - [conn205880] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:49:05.963Z I CONTROL [conn205880] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:49:25.971Z I - [conn212596] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:49:25.980Z I CONTROL [conn212596] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:49:45.963Z I - [conn205941] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:49:45.972Z I CONTROL [conn205941] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:50:05.953Z I - [conn205594] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:50:05.962Z I CONTROL [conn205594] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:50:25.968Z I - [conn212600] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:50:25.978Z I CONTROL [conn212600] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:50:45.944Z I - [conn205598] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:50:45.954Z I CONTROL [conn205598] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:51:05.951Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:51:05.962Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:51:25.951Z I - [conn216146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:51:25.960Z I CONTROL [conn216146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:51:45.961Z I - [conn216144] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:51:45.971Z I CONTROL [conn216144] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:52:05.944Z I - [conn205824] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:52:05.954Z I CONTROL [conn205824] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:52:25.943Z I - [conn205938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:52:25.953Z I CONTROL [conn205938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:52:45.954Z I - [conn206261] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:52:45.963Z I CONTROL [conn206261] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:53:05.954Z I - [conn205500] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:53:05.963Z I CONTROL [conn205500] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:53:25.962Z I - [conn205824] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:53:25.972Z I CONTROL [conn205824] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:53:45.970Z I - [conn205943] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:53:45.980Z I CONTROL [conn205943] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:54:05.951Z I - [conn216148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:54:05.960Z I CONTROL [conn216148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:54:25.962Z I - [conn205656] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:54:25.971Z I CONTROL [conn205656] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:54:45.942Z I - [conn205691] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:54:45.951Z I CONTROL [conn205691] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:55:05.940Z I - [conn215961] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:55:05.950Z I CONTROL [conn215961] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:55:25.954Z I - [conn205857] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:55:25.963Z I CONTROL [conn205857] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:55:45.955Z I - [conn205908] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:55:45.965Z I CONTROL [conn205908] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:56:05.940Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:56:05.950Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:56:25.953Z I - [conn216062] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:56:25.963Z I CONTROL [conn216062] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:56:45.953Z I - [conn216107] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:56:45.963Z I CONTROL [conn216107] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:57:05.949Z I - [conn215919] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:57:05.959Z I CONTROL [conn215919] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:57:25.950Z I - [conn207410] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:57:25.960Z I CONTROL [conn207410] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:57:45.970Z I - [conn215148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:57:45.980Z I CONTROL [conn215148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:58:05.959Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:58:05.969Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:58:25.961Z I - [conn170473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:58:25.970Z I CONTROL [conn170473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:58:45.954Z I - [conn205486] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:58:45.963Z I CONTROL [conn205486] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:59:05.945Z I - [conn215958] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:59:05.954Z I CONTROL [conn215958] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:59:25.944Z I - [conn205868] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:59:25.954Z I CONTROL [conn205868] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:59:45.946Z I - [conn216154] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-28T23:59:45.955Z I CONTROL [conn216154] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-28T23:59:48.551Z I INDEX [conn215568] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-29T00:00:05.957Z I - [conn215963] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:00:05.967Z I CONTROL [conn215963] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:00:25.952Z I - [conn205595] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:00:25.962Z I CONTROL [conn205595] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:00:45.952Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:00:45.962Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:01:05.948Z I - [conn215971] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:01:05.957Z I CONTROL [conn215971] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:01:25.955Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:01:25.965Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:01:45.959Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:01:45.969Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:02:05.947Z I - [conn205791] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:02:05.957Z I CONTROL [conn205791] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:02:25.951Z I - [conn205920] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:02:25.961Z I CONTROL [conn205920] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:02:45.955Z I - [conn205857] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:02:45.965Z I CONTROL [conn205857] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:03:05.937Z I - [conn215922] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:03:05.947Z I CONTROL [conn215922] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:03:25.951Z I - [conn215850] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:03:25.961Z I CONTROL [conn215850] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:03:45.951Z I - [conn215140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:03:45.961Z I CONTROL [conn215140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:04:05.937Z I - [conn216110] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:04:05.946Z I CONTROL [conn216110] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:04:25.946Z I - [conn205798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:04:25.956Z I CONTROL [conn205798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:04:45.963Z I - [conn216102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:04:45.972Z I CONTROL [conn216102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:05:05.957Z I - [conn205566] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:05:05.967Z I CONTROL [conn205566] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:05:25.957Z I - [conn215965] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:05:25.967Z I CONTROL [conn215965] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:05:45.947Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:05:45.957Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:06:05.936Z I - [conn205828] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:06:05.946Z I CONTROL [conn205828] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:06:25.949Z I - [conn215963] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:06:25.959Z I CONTROL [conn215963] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:06:45.954Z I - [conn215949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:06:45.965Z I CONTROL [conn215949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:07:05.949Z I - [conn207411] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:07:05.958Z I CONTROL [conn207411] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:07:25.950Z I - [conn206260] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:07:25.960Z I CONTROL [conn206260] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:07:45.933Z I - [conn205585] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:07:45.943Z I CONTROL [conn205585] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:08:05.946Z I - [conn205550] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:08:05.956Z I CONTROL [conn205550] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:08:25.947Z I - [conn205798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:08:25.956Z I CONTROL [conn205798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:08:45.946Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:08:45.955Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:09:05.934Z I - [conn205570] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:09:05.944Z I CONTROL [conn205570] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:09:25.949Z I - [conn206321] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:09:25.959Z I CONTROL [conn206321] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:09:45.933Z I - [conn215149] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:09:45.943Z I CONTROL [conn215149] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:10:05.932Z I - [conn215960] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:10:05.942Z I CONTROL [conn215960] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:10:25.943Z I - [conn205457] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:10:25.953Z I CONTROL [conn205457] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:10:45.964Z I - [conn205843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:10:45.974Z I CONTROL [conn205843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:11:05.946Z I - [conn206305] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:11:05.956Z I CONTROL [conn206305] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:11:25.940Z I - [conn215971] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:11:25.949Z I CONTROL [conn215971] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:11:45.942Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:11:45.952Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:12:05.944Z I - [conn205730] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:12:05.954Z I CONTROL [conn205730] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:12:25.944Z I - [conn205500] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:12:25.953Z I CONTROL [conn205500] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:12:45.943Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:12:45.953Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:13:05.941Z I - [conn215141] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:13:05.951Z I CONTROL [conn215141] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:13:25.946Z I - [conn206795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:13:25.956Z I CONTROL [conn206795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:13:45.943Z I - [conn183343] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:13:45.952Z I CONTROL [conn183343] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:14:05.934Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:14:05.943Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:14:25.931Z I - [conn205512] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:14:25.941Z I CONTROL [conn205512] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:14:45.952Z I - [conn206331] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:14:45.962Z I CONTROL [conn206331] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:15:05.961Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:15:05.971Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:15:26.327Z I - [conn216082] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:15:26.337Z I CONTROL [conn216082] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:15:45.938Z I - [conn215856] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:15:45.948Z I CONTROL [conn215856] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:16:05.950Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:16:05.959Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:16:25.945Z I - [conn215859] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:16:25.955Z I CONTROL [conn215859] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:16:45.940Z I - [conn215982] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:16:45.950Z I CONTROL [conn215982] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:17:05.944Z I - [conn205487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:17:05.954Z I CONTROL [conn205487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:17:25.940Z I - [conn215875] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:17:25.950Z I CONTROL [conn215875] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:17:45.944Z I - [conn215944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:17:45.954Z I CONTROL [conn215944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:18:05.942Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:18:05.951Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:18:25.934Z I - [conn205828] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:18:25.943Z I CONTROL [conn205828] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:18:37.083Z I - [conn205929] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:18:37.093Z I CONTROL [conn205929] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:18:45.941Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:18:45.950Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:19:05.946Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:19:05.956Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:19:25.941Z I - [conn216121] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:19:25.951Z I CONTROL [conn216121] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:19:45.938Z I - [conn205678] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:19:45.947Z I CONTROL [conn205678] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:20:05.939Z I - [conn216083] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:20:05.949Z I CONTROL [conn216083] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:20:25.938Z I - [conn205460] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:20:25.948Z I CONTROL [conn205460] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:20:45.945Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:20:45.955Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:21:05.934Z I - [conn183343] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:21:05.944Z I CONTROL [conn183343] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:21:25.937Z I - [conn205714] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:21:25.946Z I CONTROL [conn205714] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:21:45.944Z I - [conn205714] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:21:45.954Z I CONTROL [conn205714] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:22:05.945Z I - [conn205880] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:22:05.955Z I CONTROL [conn205880] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:22:25.926Z I - [conn170812] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:22:25.935Z I CONTROL [conn170812] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:22:45.938Z I - [conn207415] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:22:45.947Z I CONTROL [conn207415] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:23:05.947Z I - [conn216140] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:23:05.957Z I CONTROL [conn216140] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:23:25.941Z I - [conn206810] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:23:25.951Z I CONTROL [conn206810] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:23:45.925Z I - [conn205566] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:23:45.935Z I CONTROL [conn205566] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:24:05.939Z I - [conn205904] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:24:05.949Z I CONTROL [conn205904] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:24:25.948Z I - [conn216059] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:24:25.958Z I CONTROL [conn216059] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:24:45.936Z I - [conn205939] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:24:45.946Z I CONTROL [conn205939] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:25:05.943Z I - [conn215148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:25:05.953Z I CONTROL [conn215148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:25:25.958Z I - [conn205881] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:25:25.967Z I CONTROL [conn205881] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:25:45.949Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:25:45.959Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:26:05.938Z I - [conn183344] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:26:05.948Z I CONTROL [conn183344] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:26:25.939Z I - [conn216112] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:26:25.948Z I CONTROL [conn216112] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:26:45.949Z I - [conn207685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:26:45.959Z I CONTROL [conn207685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:27:05.938Z I - [conn216053] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:27:05.947Z I CONTROL [conn216053] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:27:25.923Z I - [conn216053] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:27:25.932Z I CONTROL [conn216053] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:27:45.935Z I - [conn206260] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:27:45.944Z I CONTROL [conn206260] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:28:05.934Z I - [conn206769] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:28:05.943Z I CONTROL [conn206769] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:28:25.932Z I - [conn215938] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:28:25.941Z I CONTROL [conn215938] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:28:45.935Z I - [conn206255] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:28:45.944Z I CONTROL [conn206255] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:29:05.934Z I - [conn216143] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:29:05.943Z I CONTROL [conn216143] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:29:25.934Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:29:25.943Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:29:45.937Z I - [conn215146] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:29:45.946Z I CONTROL [conn215146] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:30:05.945Z I - [conn177378] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:30:05.954Z I CONTROL [conn177378] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:30:25.931Z I - [conn216122] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:30:25.941Z I CONTROL [conn216122] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:30:45.932Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:30:45.942Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:31:05.933Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:31:05.943Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:31:25.920Z I - [conn215955] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:31:25.930Z I CONTROL [conn215955] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:31:45.936Z I - [conn206332] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:31:45.945Z I CONTROL [conn206332] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:32:05.939Z I - [conn216081] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:32:05.948Z I CONTROL [conn216081] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:32:25.939Z I - [conn205659] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:32:25.949Z I CONTROL [conn205659] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:32:45.920Z I - [conn216155] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:32:45.930Z I CONTROL [conn216155] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:33:05.927Z I - [conn216155] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:33:05.937Z I CONTROL [conn216155] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:33:25.922Z I - [conn206315] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:33:25.932Z I CONTROL [conn206315] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:33:45.924Z I - [conn207414] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:33:45.934Z I CONTROL [conn207414] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:34:05.929Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:34:05.939Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:34:25.933Z I - [conn170471] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:34:25.943Z I CONTROL [conn170471] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:34:45.935Z I - [conn205948] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:34:45.945Z I CONTROL [conn205948] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:34:51.926Z I - [conn216102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:34:51.936Z I CONTROL [conn216102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:35:05.932Z I - [conn216102] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:35:05.941Z I CONTROL [conn216102] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:35:25.930Z I - [conn206768] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:35:25.940Z I CONTROL [conn206768] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:35:45.933Z I - [conn216148] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:35:45.943Z I CONTROL [conn216148] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:36:05.928Z I - [conn205566] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:36:05.938Z I CONTROL [conn205566] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:36:25.921Z I - [conn205798] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:36:25.931Z I CONTROL [conn205798] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:36:45.925Z I - [conn216135] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:36:45.935Z I CONTROL [conn216135] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:37:05.918Z I - [conn177378] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:37:05.928Z I CONTROL [conn177378] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:37:25.932Z I - [conn215940] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:37:25.942Z I CONTROL [conn215940] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:37:45.926Z I - [conn205740] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:37:45.935Z I CONTROL [conn205740] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:38:05.928Z I - [conn215843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:38:05.938Z I CONTROL [conn215843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:38:25.946Z I - [conn205518] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:38:25.956Z I CONTROL [conn205518] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:38:45.930Z I - [conn215958] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:38:45.940Z I CONTROL [conn215958] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:39:05.923Z I - [conn205607] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:39:05.932Z I CONTROL [conn205607] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:39:25.937Z I - [conn205685] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:39:25.947Z I CONTROL [conn205685] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:39:45.929Z I - [conn205569] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:39:45.938Z I CONTROL [conn205569] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:40:05.928Z I - [conn215914] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:40:05.938Z I CONTROL [conn215914] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:40:25.926Z I - [conn205778] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:40:25.936Z I CONTROL [conn205778] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:40:45.933Z I - [conn177440] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:40:45.942Z I CONTROL [conn177440] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:41:05.927Z I - [conn206320] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:41:05.936Z I CONTROL [conn206320] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:41:25.923Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:41:25.932Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:41:45.927Z I - [conn207402] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:41:45.937Z I CONTROL [conn207402] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:42:05.935Z I - [conn216133] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:42:05.945Z I CONTROL [conn216133] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:42:25.920Z I - [conn212230] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:42:25.929Z I CONTROL [conn212230] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:42:45.928Z I - [conn205736] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:42:45.938Z I CONTROL [conn205736] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:43:05.924Z I - [conn216112] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:43:05.934Z I CONTROL [conn216112] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:43:25.915Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:43:25.924Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:43:45.926Z I - [conn205925] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:43:45.936Z I CONTROL [conn205925] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:44:05.930Z I - [conn205795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:44:05.940Z I CONTROL [conn205795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:44:25.925Z I - [conn205550] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:44:25.935Z I CONTROL [conn205550] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:44:45.933Z I - [conn215981] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:44:45.943Z I CONTROL [conn215981] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:45:05.928Z I - [conn205949] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:45:05.938Z I CONTROL [conn205949] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:45:25.925Z I - [conn205928] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:45:25.935Z I CONTROL [conn205928] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:45:45.924Z I - [conn215937] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:45:45.934Z I CONTROL [conn215937] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:46:05.928Z I - [conn205457] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:46:05.938Z I CONTROL [conn205457] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:46:25.937Z I - [conn177448] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:46:25.947Z I CONTROL [conn177448] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:46:45.926Z I - [conn205498] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:46:45.935Z I CONTROL [conn205498] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:47:05.918Z I - [conn215972] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:47:05.928Z I CONTROL [conn215972] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:47:25.925Z I - [conn177807] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:47:25.934Z I CONTROL [conn177807] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:47:45.922Z I - [conn216063] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:47:45.932Z I CONTROL [conn216063] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:48:05.922Z I - [conn205760] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:48:05.932Z I CONTROL [conn205760] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:48:25.912Z I - [conn183099] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:48:25.922Z I CONTROL [conn183099] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:48:45.929Z I - [conn216122] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:48:45.939Z I CONTROL [conn216122] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:49:05.924Z I - [conn205577] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:49:05.934Z I CONTROL [conn205577] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:49:25.922Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:49:25.932Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:49:45.920Z I - [conn205795] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:49:45.929Z I CONTROL [conn205795] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:50:05.931Z I - [conn215976] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:50:05.940Z I CONTROL [conn215976] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:50:25.927Z I - [conn205848] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:50:25.937Z I CONTROL [conn205848] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:50:45.923Z I - [conn215755] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:50:45.933Z I CONTROL [conn215755] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:51:05.924Z I - [conn205728] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:51:05.934Z I CONTROL [conn205728] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:51:25.925Z I - [conn205550] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:51:25.935Z I CONTROL [conn205550] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:51:45.924Z I - [conn205550] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:51:45.933Z I CONTROL [conn205550] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:52:05.925Z I - [conn205843] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:52:05.936Z I CONTROL [conn205843] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:52:25.925Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:52:25.934Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:52:45.926Z I - [conn206772] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:52:45.936Z I CONTROL [conn206772] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:53:05.925Z I - [conn205516] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:53:05.935Z I CONTROL [conn205516] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:53:25.924Z I - [conn212226] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:53:25.934Z I CONTROL [conn212226] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:53:45.922Z I - [conn205872] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:53:45.931Z I CONTROL [conn205872] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:54:05.908Z I - [conn216096] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:54:05.918Z I CONTROL [conn216096] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:54:25.926Z I - [conn207411] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:54:25.935Z I CONTROL [conn207411] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:54:45.908Z I - [conn216096] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:54:45.917Z I CONTROL [conn216096] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:55:05.931Z I - [conn205728] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:55:05.941Z I CONTROL [conn205728] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:55:25.911Z I - [conn205809] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:55:25.921Z I CONTROL [conn205809] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:55:45.919Z I - [conn215912] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:55:45.930Z I CONTROL [conn215912] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:56:05.931Z I - [conn205881] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:56:05.941Z I CONTROL [conn205881] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:56:25.927Z I - [conn177442] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:56:25.937Z I CONTROL [conn177442] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:56:45.909Z I - [conn205518] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:56:45.919Z I CONTROL [conn205518] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:57:05.908Z I - [conn205867] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:57:05.918Z I CONTROL [conn205867] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:57:25.923Z I - [conn205683] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:57:25.932Z I CONTROL [conn205683] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:57:45.907Z I - [conn216065] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:57:45.916Z I CONTROL [conn216065] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:58:05.906Z I - [conn215960] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:58:05.916Z I CONTROL [conn215960] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:58:25.919Z I - [conn206298] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:58:25.928Z I CONTROL [conn206298] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:58:45.920Z I - [conn205659] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:58:45.929Z I CONTROL [conn205659] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:59:05.918Z I - [conn205500] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:59:05.928Z I CONTROL [conn205500] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:59:25.910Z I - [conn170504] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:59:25.920Z I CONTROL [conn170504] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:59:45.920Z I - [conn206800] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T00:59:45.930Z I CONTROL [conn206800] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T00:59:48.499Z I INDEX [conn215568] build index on: sledge.sophos_labs_threats properties: { v: 1, unique: true, key: { name: 1, threat_type: 1 }, name: "name_1_threat_type_1", ns: "sledge.sophos_labs_threats", background: true } 2015-10-29T01:00:05.946Z I - [conn206800] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:00:05.956Z I CONTROL [conn206800] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:00:25.907Z I - [conn205796] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:00:25.917Z I CONTROL [conn205796] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:00:45.929Z I - [conn205496] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:00:45.939Z I CONTROL [conn205496] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:01:05.919Z I - [conn215965] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:01:05.929Z I CONTROL [conn215965] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:01:25.907Z I - [conn205579] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:01:25.917Z I CONTROL [conn205579] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:01:45.920Z I - [conn215874] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:01:45.929Z I CONTROL [conn215874] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:02:05.920Z I - [conn205831] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:02:05.930Z I CONTROL [conn205831] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:02:25.922Z I - [conn205626] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:02:25.931Z I CONTROL [conn205626] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:02:45.927Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:02:45.937Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:03:05.919Z I - [conn206312] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:03:05.928Z I CONTROL [conn206312] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:03:25.924Z I - [conn205768] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:03:25.935Z I CONTROL [conn205768] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:03:45.919Z I - [conn177448] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:03:45.931Z I CONTROL [conn177448] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:04:05.918Z I - [conn207417] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:04:05.927Z I CONTROL [conn207417] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:04:25.904Z I - [conn183099] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:04:25.914Z I CONTROL [conn183099] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:04:45.917Z I - [conn216135] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:04:45.928Z I CONTROL [conn216135] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:05:05.907Z I - [conn183099] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:05:05.917Z I CONTROL [conn183099] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:05:25.916Z I - [conn205594] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:05:25.926Z I CONTROL [conn205594] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:05:45.906Z I - [conn216159] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:05:45.915Z I CONTROL [conn216159] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:06:05.915Z I - [conn215944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:06:05.925Z I CONTROL [conn215944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:06:25.901Z I - [conn215944] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:06:25.911Z I CONTROL [conn215944] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:06:45.905Z I - [conn205904] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:06:45.914Z I CONTROL [conn205904] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:07:05.920Z I - [conn216089] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:07:05.929Z I CONTROL [conn216089] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:07:25.916Z I - [conn183345] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:07:25.927Z I CONTROL [conn183345] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:07:45.918Z I - [conn205573] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:07:45.928Z I CONTROL [conn205573] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:08:05.919Z I - [conn183429] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:08:05.929Z I CONTROL [conn183429] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:08:25.916Z I - [conn205740] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:08:25.925Z I CONTROL [conn205740] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:08:45.908Z I - [conn205816] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:08:45.918Z I CONTROL [conn205816] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:09:05.913Z I - [conn205582] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:09:05.923Z I CONTROL [conn205582] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:09:25.915Z I - [conn215969] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:09:25.925Z I CONTROL [conn215969] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:09:45.919Z I - [conn207407] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:09:45.929Z I CONTROL [conn207407] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:10:05.913Z I - [conn205594] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:10:05.923Z I CONTROL [conn205594] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:10:25.919Z I - [conn205829] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:10:25.929Z I CONTROL [conn205829] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:10:45.903Z I - [conn183429] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:10:45.912Z I CONTROL [conn183429] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:11:05.901Z I - [conn205659] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:11:05.912Z I CONTROL [conn205659] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:11:25.918Z I - [conn205618] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:11:25.927Z I CONTROL [conn205618] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:11:45.913Z I - [conn205707] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:11:45.922Z I CONTROL [conn205707] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:12:05.918Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:12:05.927Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:12:25.913Z I - [conn205776] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:12:25.922Z I CONTROL [conn205776] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:12:45.920Z I - [conn215961] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:12:45.930Z I CONTROL [conn215961] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:13:05.907Z I - [conn205492] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:13:05.916Z I CONTROL [conn205492] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:13:25.900Z I - [conn205848] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:13:25.910Z I CONTROL [conn205848] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:13:33.897Z I - [conn225342] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:13:33.905Z I CONTROL [conn225342] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:13:33.905Z I QUERY [conn225342] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-29T01:13:45.904Z I - [conn205487] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:13:45.914Z I CONTROL [conn205487] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:14:05.903Z I - [conn205618] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:14:05.913Z I CONTROL [conn205618] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:14:25.916Z I - [conn216080] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:14:25.925Z I CONTROL [conn216080] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:14:45.913Z I - [conn205671] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:14:45.923Z I CONTROL [conn205671] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:15:05.910Z I - [conn205791] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:15:05.919Z I CONTROL [conn205791] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:15:25.904Z I - [conn205854] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:15:25.914Z I CONTROL [conn205854] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:15:45.922Z I - [conn206794] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:15:45.932Z I CONTROL [conn206794] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:16:05.919Z I - [conn183342] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:16:05.928Z I CONTROL [conn183342] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:16:25.920Z I - [conn215906] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:16:25.929Z I CONTROL [conn215906] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:16:45.902Z I - [conn205853] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:16:45.911Z I CONTROL [conn205853] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:17:05.922Z I - [conn207413] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:17:05.931Z I CONTROL [conn207413] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:17:25.908Z I - [conn171215] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:17:25.917Z I CONTROL [conn171215] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:17:45.927Z I - [conn205473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:17:45.937Z I CONTROL [conn205473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:18:05.906Z I - [conn205473] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:18:05.916Z I CONTROL [conn205473] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:18:25.897Z I - [conn205882] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:18:25.906Z I CONTROL [conn205882] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:18:45.910Z I - [conn215142] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:18:45.920Z I CONTROL [conn215142] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:19:05.910Z I - [conn205422] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:19:05.920Z I CONTROL [conn205422] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:19:25.910Z I - [conn205567] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:19:25.921Z I CONTROL [conn205567] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:19:45.906Z I - [conn215913] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:19:45.915Z I CONTROL [conn215913] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:20:05.907Z I - [conn216123] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:20:05.916Z I CONTROL [conn216123] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:20:12.585Z I - [conn225373] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:20:12.593Z I CONTROL [conn225373] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:20:12.593Z I QUERY [conn225373] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-29T01:20:25.897Z I - [conn216120] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:20:25.907Z I CONTROL [conn216120] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:20:28.349Z I - [conn225373] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:20:28.381Z I CONTROL [conn225373] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BDC03"},{"b":"400000","o":"5CD409"},{"b":"400000","o":"5CDB1B"},{"b":"400000","o":"5CE1CB"},{"b":"400000","o":"5D0B7D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo17getExecutorDeleteEPNS_16OperationContextEPNS_10CollectionEPNS_12ParsedDeleteEPPNS_12PlanExecutorE+0x833) [0xbbdc03] mongod(_ZN5mongo18WriteBatchExecutor10execRemoveERKNS_12BatchItemRefEPPNS_16WriteErrorDetailE+0x469) [0x9cd409] mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaISE_EE+0xCB) [0x9cdb1b] mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x37B) [0x9ce1cb] mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x15D) [0x9d0b7d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:20:35.237Z I - [conn225373] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:20:35.245Z I CONTROL [conn225373] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:20:35.245Z I QUERY [conn225373] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-29T01:20:45.899Z I - [conn205633] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:20:45.910Z I CONTROL [conn205633] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:21:05.907Z I - [conn216065] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:21:05.916Z I CONTROL [conn216065] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7C0AFA"},{"b":"400000","o":"574F8D"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo19getExecutorDistinctEPNS_16OperationContextEPNS_10CollectionERKNS_7BSONObjERKSsNS_12PlanExecutor11YieldPolicyEPPS9_+0x13AA) [0xbc0afa] mongod(_ZN5mongo15DistinctCommand3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x38D) [0x974f8d] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:21:24.011Z I REPL [ReplicationExecutor] transition to SECONDARY 2015-10-29T01:21:24.013Z I QUERY [conn205529] assertion 13435 not master and slaveOk=false ns:sledge.endpoints query:{ _id: BinData(3, 3D685969761446099614D1A9245DC002) } 2015-10-29T01:21:24.021Z I NETWORK [conn205543] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52122] 2015-10-29T01:21:24.021Z I NETWORK [conn205901] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55343] 2015-10-29T01:21:24.021Z I NETWORK [conn205929] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37297] 2015-10-29T01:21:24.021Z I NETWORK [conn206321] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35381] 2015-10-29T01:21:24.021Z I NETWORK [conn205664] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55295] 2015-10-29T01:21:24.021Z I NETWORK [conn215984] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50800] 2015-10-29T01:21:24.021Z I NETWORK [conn215940] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50756] 2015-10-29T01:21:24.021Z I NETWORK [conn183343] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:48071] 2015-10-29T01:21:24.022Z I NETWORK [conn206254] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:34960] 2015-10-29T01:21:24.022Z I NETWORK [conn215907] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50723] 2015-10-29T01:21:24.022Z I NETWORK [conn216125] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53646] 2015-10-29T01:21:24.022Z I NETWORK [conn183335] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:36743] 2015-10-29T01:21:24.022Z I NETWORK [conn215975] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50791] 2015-10-29T01:21:24.021Z I NETWORK [conn206773] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39398] 2015-10-29T01:21:24.023Z I NETWORK [conn206774] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39399] 2015-10-29T01:21:24.022Z I NETWORK [conn216132] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53653] 2015-10-29T01:21:24.022Z I NETWORK [conn205825] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52160] 2015-10-29T01:21:24.023Z I NETWORK [conn215854] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50697] 2015-10-29T01:21:24.023Z I NETWORK [conn205816] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55320] 2015-10-29T01:21:24.022Z I NETWORK [conn205631] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37249] 2015-10-29T01:21:24.024Z I NETWORK [conn216146] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53667] 2015-10-29T01:21:24.022Z I NETWORK [conn205942] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37305] 2015-10-29T01:21:24.021Z I NETWORK [conn206329] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35429] 2015-10-29T01:21:24.023Z I NETWORK [conn171044] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:45523] 2015-10-29T01:21:24.024Z I NETWORK [conn216052] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53437] 2015-10-29T01:21:24.023Z I NETWORK [conn205679] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52136] 2015-10-29T01:21:24.023Z I NETWORK [conn206808] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39559] 2015-10-29T01:21:24.025Z I NETWORK [conn215849] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50692] 2015-10-29T01:21:24.025Z I NETWORK [conn205685] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55301] 2015-10-29T01:21:24.026Z I NETWORK [conn216107] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53628] 2015-10-29T01:21:24.026Z I NETWORK [conn205457] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37125] 2015-10-29T01:21:24.026Z I NETWORK [conn216161] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53682] 2015-10-29T01:21:24.026Z I NETWORK [conn215946] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50762] 2015-10-29T01:21:24.026Z I NETWORK [conn205582] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55279] 2015-10-29T01:21:24.027Z I NETWORK [conn216100] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53621] 2015-10-29T01:21:24.027Z I NETWORK [conn205920] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52175] 2015-10-29T01:21:24.028Z I NETWORK [conn216151] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53672] 2015-10-29T01:21:24.024Z I NETWORK [conn205829] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37282] 2015-10-29T01:21:24.028Z I NETWORK [conn205765] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37273] 2015-10-29T01:21:24.028Z I NETWORK [conn205486] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55158] 2015-10-29T01:21:24.028Z I NETWORK [conn215977] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50793] 2015-10-29T01:21:24.024Z I NETWORK [conn170471] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:58496] 2015-10-29T01:21:24.029Z I NETWORK [conn215976] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50792] 2015-10-29T01:21:24.029Z I NETWORK [conn215973] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50789] 2015-10-29T01:21:24.030Z I NETWORK [conn183171] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:46405] 2015-10-29T01:21:24.030Z I NETWORK [conn205606] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55283] 2015-10-29T01:21:24.030Z I NETWORK [conn205567] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37237] 2015-10-29T01:21:24.030Z I NETWORK [conn215855] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50699] 2015-10-29T01:21:24.025Z I NETWORK [conn212230] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:33876] 2015-10-29T01:21:24.030Z I NETWORK [conn183429] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:58641] 2015-10-29T01:21:24.030Z I NETWORK [conn205857] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52166] 2015-10-29T01:21:24.023Z I NETWORK [conn205465] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:51966] 2015-10-29T01:21:24.031Z I NETWORK [conn205814] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52158] 2015-10-29T01:21:24.031Z I NETWORK [conn215142] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:60731] 2015-10-29T01:21:24.031Z I NETWORK [conn216154] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53675] 2015-10-29T01:21:24.031Z I NETWORK [conn215911] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50727] 2015-10-29T01:21:24.025Z I NETWORK [conn215958] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50774] 2015-10-29T01:21:24.032Z I NETWORK [conn215912] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50728] 2015-10-29T01:21:24.032Z I NETWORK [conn216133] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53654] 2015-10-29T01:21:24.032Z I NETWORK [conn205872] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55338] 2015-10-29T01:21:24.032Z I NETWORK [conn205826] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55324] 2015-10-29T01:21:24.025Z I NETWORK [conn215960] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50776] 2015-10-29T01:21:24.033Z I NETWORK [conn205811] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52157] 2015-10-29T01:21:24.033Z I NETWORK [conn216081] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53572] 2015-10-29T01:21:24.033Z I NETWORK [conn205682] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55300] 2015-10-29T01:21:24.034Z I NETWORK [conn205518] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52115] 2015-10-29T01:21:24.034Z I NETWORK [conn183102] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:55510] 2015-10-29T01:21:24.026Z I NETWORK [conn205517] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52114] 2015-10-29T01:21:24.035Z I NETWORK [conn215947] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50763] 2015-10-29T01:21:24.023Z I NETWORK [conn215141] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:60730] 2015-10-29T01:21:24.035Z I NETWORK [conn205479] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:51970] 2015-10-29T01:21:24.026Z I NETWORK [conn205774] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37274] 2015-10-29T01:21:24.035Z I NETWORK [conn207402] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45147] 2015-10-29T01:21:24.024Z I NETWORK [conn206769] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39394] 2015-10-29T01:21:24.035Z I NETWORK [conn216123] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53644] 2015-10-29T01:21:24.035Z I NETWORK [conn205692] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52139] 2015-10-29T01:21:24.036Z I NETWORK [conn206772] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39397] 2015-10-29T01:21:24.036Z I NETWORK [conn216062] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53549] 2015-10-29T01:21:24.036Z I NETWORK [conn216108] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53629] 2015-10-29T01:21:24.027Z I NETWORK [conn216128] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53649] 2015-10-29T01:21:24.036Z I NETWORK [conn215970] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50786] 2015-10-29T01:21:24.036Z I NETWORK [conn216155] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53676] 2015-10-29T01:21:24.022Z I NETWORK [conn215936] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50752] 2015-10-29T01:21:24.036Z I NETWORK [conn215851] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50694] 2015-10-29T01:21:24.027Z I NETWORK [conn177467] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:54685] 2015-10-29T01:21:24.037Z I NETWORK [conn205743] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52143] 2015-10-29T01:21:24.037Z I NETWORK [conn207685] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:47436] 2015-10-29T01:21:24.037Z I NETWORK [conn170898] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:44041] 2015-10-29T01:21:24.037Z I NETWORK [conn207415] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45160] 2015-10-29T01:21:24.024Z I NETWORK [conn216127] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53648] 2015-10-29T01:21:24.037Z I NETWORK [conn205903] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37293] 2015-10-29T01:21:24.028Z I NETWORK [conn205601] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55282] 2015-10-29T01:21:24.037Z I NETWORK [conn205849] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52164] 2015-10-29T01:21:24.038Z I NETWORK [conn206327] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35427] 2015-10-29T01:21:24.038Z I NETWORK [conn206298] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35276] 2015-10-29T01:21:24.038Z I NETWORK [conn205939] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52186] 2015-10-29T01:21:24.023Z I NETWORK [conn205749] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55304] 2015-10-29T01:21:24.028Z I NETWORK [conn207407] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45152] 2015-10-29T01:21:24.029Z I NETWORK [conn215877] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50716] 2015-10-29T01:21:24.029Z I NETWORK [conn212598] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:37377] 2015-10-29T01:21:24.039Z I NETWORK [conn205869] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55337] 2015-10-29T01:21:24.029Z I NETWORK [conn207409] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45154] 2015-10-29T01:21:24.029Z I NETWORK [conn177378] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:32866] 2015-10-29T01:21:24.023Z I NETWORK [conn225373] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [127.0.0.1:55798] 2015-10-29T01:21:24.040Z I NETWORK [conn215925] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50741] 2015-10-29T01:21:24.029Z I NETWORK [conn212601] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:37381] 2015-10-29T01:21:24.040Z I NETWORK [conn212599] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:37379] 2015-10-29T01:21:24.040Z I NETWORK [conn170812] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:50344] 2015-10-29T01:21:24.040Z I NETWORK [conn205924] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52176] 2015-10-29T01:21:24.040Z I NETWORK [conn216156] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53677] 2015-10-29T01:21:24.040Z I NETWORK [conn215906] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50722] 2015-10-29T01:21:24.041Z I NETWORK [conn205722] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37266] 2015-10-29T01:21:24.041Z I NETWORK [conn216102] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53623] 2015-10-29T01:21:24.041Z I NETWORK [conn216099] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53620] 2015-10-29T01:21:24.041Z I NETWORK [conn207406] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45151] 2015-10-29T01:21:24.041Z I NETWORK [conn207414] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45159] 2015-10-29T01:21:24.041Z I NETWORK [conn205796] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52152] 2015-10-29T01:21:24.031Z I NETWORK [conn206799] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39479] 2015-10-29T01:21:24.042Z I NETWORK [conn205904] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37294] 2015-10-29T01:21:24.031Z I NETWORK [conn205745] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37271] 2015-10-29T01:21:24.025Z I NETWORK [conn205712] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37263] 2015-10-29T01:21:24.022Z I NETWORK [conn216110] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53631] 2015-10-29T01:21:24.042Z I NETWORK [conn206795] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39475] 2015-10-29T01:21:24.032Z I NETWORK [conn215966] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50782] 2015-10-29T01:21:24.042Z I NETWORK [conn205918] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52174] 2015-10-29T01:21:24.042Z I NETWORK [conn205487] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:51972] 2015-10-29T01:21:24.043Z I NETWORK [conn205945] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52190] 2015-10-29T01:21:24.032Z I NETWORK [conn216089] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53610] 2015-10-29T01:21:24.023Z I NETWORK [conn205659] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37253] 2015-10-29T01:21:24.033Z I NETWORK [conn205853] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52165] 2015-10-29T01:21:24.043Z I NETWORK [conn205787] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55313] 2015-10-29T01:21:24.033Z I NETWORK [conn205948] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52194] 2015-10-29T01:21:24.033Z I NETWORK [conn205566] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37236] 2015-10-29T01:21:24.022Z I NETWORK [conn205615] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52130] 2015-10-29T01:21:24.033Z I NETWORK [conn212225] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:33871] 2015-10-29T01:21:24.044Z I NETWORK [conn215874] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50713] 2015-10-29T01:21:24.033Z I NETWORK [conn205925] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52177] 2015-10-29T01:21:24.044Z I NETWORK [conn215140] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:60729] 2015-10-29T01:21:24.034Z I NETWORK [conn206316] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35376] 2015-10-29T01:21:24.023Z I NETWORK [conn205617] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37247] 2015-10-29T01:21:24.044Z I NETWORK [conn205949] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52195] 2015-10-29T01:21:24.044Z I NETWORK [conn215965] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50781] 2015-10-29T01:21:24.045Z I NETWORK [conn215852] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50695] 2015-10-29T01:21:24.045Z I NETWORK [conn205570] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37239] 2015-10-29T01:21:24.034Z I NETWORK [conn205815] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37281] 2015-10-29T01:21:24.034Z I NETWORK [conn207412] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45157] 2015-10-29T01:21:24.045Z I NETWORK [conn205638] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52133] 2015-10-29T01:21:24.045Z I NETWORK [conn205756] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52145] 2015-10-29T01:21:24.045Z I NETWORK [conn205809] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52156] 2015-10-29T01:21:24.024Z I NETWORK [conn205656] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55293] 2015-10-29T01:21:24.035Z I NETWORK [conn216093] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53614] 2015-10-29T01:21:24.046Z I NETWORK [conn183115] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:34603] 2015-10-29T01:21:24.035Z I NETWORK [conn215146] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:60736] 2015-10-29T01:21:24.046Z I NETWORK [conn216145] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53666] 2015-10-29T01:21:24.036Z I NETWORK [conn215913] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50729] 2015-10-29T01:21:24.046Z I NETWORK [conn205813] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55319] 2015-10-29T01:21:24.022Z I NETWORK [conn205666] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52134] 2015-10-29T01:21:24.027Z I NETWORK [conn206256] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:34966] 2015-10-29T01:21:24.024Z I NETWORK [conn205633] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52132] 2015-10-29T01:21:24.047Z I NETWORK [conn177404] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:54109] 2015-10-29T01:21:24.047Z I NETWORK [conn205867] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37286] 2015-10-29T01:21:24.047Z I NETWORK [conn206328] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35428] 2015-10-29T01:21:24.047Z I NETWORK [conn206809] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39560] 2015-10-29T01:21:24.036Z I NETWORK [conn171215] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37496] 2015-10-29T01:21:24.047Z I NETWORK [conn215932] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50748] 2015-10-29T01:21:24.037Z I NETWORK [conn205941] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37304] 2015-10-29T01:21:24.027Z I NETWORK [conn205464] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:51965] 2015-10-29T01:21:24.037Z I NETWORK [conn215931] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50747] 2015-10-29T01:21:24.037Z I NETWORK [conn205824] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55323] 2015-10-29T01:21:24.048Z I NETWORK [conn216129] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53650] 2015-10-29T01:21:24.048Z I NETWORK [conn216131] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53652] 2015-10-29T01:21:24.048Z I NETWORK [conn205460] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37126] 2015-10-29T01:21:24.037Z I NETWORK [conn177448] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:44842] 2015-10-29T01:21:24.037Z I NETWORK [conn205569] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37238] 2015-10-29T01:21:24.048Z I NETWORK [conn177468] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:54684] 2015-10-29T01:21:24.037Z I NETWORK [conn205752] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55305] 2015-10-29T01:21:24.049Z I NETWORK [conn215856] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50702] 2015-10-29T01:21:24.037Z I NETWORK [conn205456] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37124] 2015-10-29T01:21:24.028Z I NETWORK [conn183103] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:55511] 2015-10-29T01:21:24.049Z I NETWORK [conn215980] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50796] 2015-10-29T01:21:24.050Z I NETWORK [conn215959] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50775] 2015-10-29T01:21:24.050Z I NETWORK [conn205598] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52127] 2015-10-29T01:21:24.050Z I NETWORK [conn205714] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37264] 2015-10-29T01:21:24.050Z I NETWORK [conn205812] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37280] 2015-10-29T01:21:24.050Z I NETWORK [conn215919] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50735] 2015-10-29T01:21:24.050Z I NETWORK [conn216142] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53663] 2015-10-29T01:21:24.028Z I NETWORK [conn215937] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50753] 2015-10-29T01:21:24.038Z I NETWORK [conn171685] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:59306] 2015-10-29T01:21:24.051Z I NETWORK [conn215950] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50766] 2015-10-29T01:21:24.051Z I NETWORK [conn206806] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39557] 2015-10-29T01:21:24.028Z I NETWORK [conn215915] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50731] 2015-10-29T01:21:24.051Z I NETWORK [conn207403] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45146] 2015-10-29T01:21:24.051Z I NETWORK [conn215969] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50785] 2015-10-29T01:21:24.024Z I NETWORK [conn205907] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55344] 2015-10-29T01:21:24.052Z I NETWORK [conn216148] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53669] 2015-10-29T01:21:24.023Z I NETWORK [conn183087] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:45693] 2015-10-29T01:21:24.024Z I NETWORK [conn206259] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:34972] 2015-10-29T01:21:24.052Z I NETWORK [conn205795] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37277] 2015-10-29T01:21:24.052Z I NETWORK [conn205791] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55315] 2015-10-29T01:21:24.052Z I NETWORK [conn215880] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50719] 2015-10-29T01:21:24.052Z I NETWORK [conn205548] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55269] 2015-10-29T01:21:24.028Z I NETWORK [conn205822] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55322] 2015-10-29T01:21:24.029Z I NETWORK [conn206797] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39477] 2015-10-29T01:21:24.039Z I NETWORK [conn215144] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:60735] 2015-10-29T01:21:24.053Z I NETWORK [conn215147] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:60737] 2015-10-29T01:21:24.053Z I NETWORK [conn216095] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53616] 2015-10-29T01:21:24.053Z I NETWORK [conn205515] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52112] 2015-10-29T01:21:24.025Z I NETWORK [conn212597] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:37378] 2015-10-29T01:21:24.053Z I NETWORK [conn205900] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37292] 2015-10-29T01:21:24.054Z I NETWORK [conn205819] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55321] 2015-10-29T01:21:24.029Z I NETWORK [conn216120] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53641] 2015-10-29T01:21:24.054Z I NETWORK [conn215948] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50764] 2015-10-29T01:21:24.040Z I NETWORK [conn205878] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37290] 2015-10-29T01:21:24.054Z I NETWORK [conn216139] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53660] 2015-10-29T01:21:24.054Z I NETWORK [conn216053] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53438] 2015-10-29T01:21:24.023Z I NETWORK [conn215938] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50754] 2015-10-29T01:21:24.023Z I NETWORK [conn205626] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52131] 2015-10-29T01:21:24.055Z I NETWORK [conn216135] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53656] 2015-10-29T01:21:24.055Z I NETWORK [conn205646] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55292] 2015-10-29T01:21:24.040Z I NETWORK [conn215135] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:60724] 2015-10-29T01:21:24.040Z I NETWORK [conn216079] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53570] 2015-10-29T01:21:24.040Z I NETWORK [conn216115] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53636] 2015-10-29T01:21:24.040Z I NETWORK [conn205764] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52147] 2015-10-29T01:21:24.055Z I NETWORK [conn216116] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53637] 2015-10-29T01:21:24.040Z I NETWORK [conn177419] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:44577] 2015-10-29T01:21:24.040Z I NETWORK [conn183336] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:36742] 2015-10-29T01:21:24.040Z I NETWORK [conn215967] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50783] 2015-10-29T01:21:24.040Z I NETWORK [conn178530] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:44509] 2015-10-29T01:21:24.040Z I NETWORK [conn205871] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37287] 2015-10-29T01:21:24.040Z I NETWORK [conn206775] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39400] 2015-10-29T01:21:24.056Z I NETWORK [conn205572] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55275] 2015-10-29T01:21:24.056Z I NETWORK [conn207411] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45156] 2015-10-29T01:21:24.040Z I NETWORK [conn205421] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55046] 2015-10-29T01:21:24.040Z I NETWORK [conn205588] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52126] 2015-10-29T01:21:24.040Z I NETWORK [conn205527] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52119] 2015-10-29T01:21:24.030Z I NETWORK [conn205856] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55332] 2015-10-29T01:21:24.041Z I NETWORK [conn205658] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55294] 2015-10-29T01:21:24.056Z I NETWORK [conn216077] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53568] 2015-10-29T01:21:24.041Z I NETWORK [conn216137] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53658] 2015-10-29T01:21:24.030Z I NETWORK [conn215923] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50739] 2015-10-29T01:21:24.041Z I NETWORK [conn215916] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50732] 2015-10-29T01:21:24.025Z I NETWORK [conn215956] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50772] 2015-10-29T01:21:24.030Z I NETWORK [conn205783] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52150] 2015-10-29T01:21:24.041Z I NETWORK [conn205451] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37120] 2015-10-29T01:21:24.057Z I NETWORK [conn183345] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:48072] 2015-10-29T01:21:24.042Z I NETWORK [conn205675] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55296] 2015-10-29T01:21:24.022Z I NETWORK [conn170779] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:42893] 2015-10-29T01:21:24.042Z I NETWORK [conn215755] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:34572] 2015-10-29T01:21:24.057Z I NETWORK [conn206318] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35378] 2015-10-29T01:21:24.057Z I NETWORK [conn206260] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:34973] 2015-10-29T01:21:24.057Z I NETWORK [conn216091] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53612] 2015-10-29T01:21:24.057Z I NETWORK [conn207410] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45155] 2015-10-29T01:21:24.032Z I NETWORK [conn205748] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55303] 2015-10-29T01:21:24.042Z I NETWORK [conn183344] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:48076] 2015-10-29T01:21:24.043Z I NETWORK [conn215981] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50797] 2015-10-29T01:21:24.058Z I NETWORK [conn205927] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37296] 2015-10-29T01:21:24.058Z I NETWORK [conn205854] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55331] 2015-10-29T01:21:24.058Z I NETWORK [conn205492] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55159] 2015-10-29T01:21:24.043Z I NETWORK [conn215955] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50771] 2015-10-29T01:21:24.043Z I NETWORK [conn212226] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:33872] 2015-10-29T01:21:24.033Z I NETWORK [conn215961] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50777] 2015-10-29T01:21:24.058Z I NETWORK [conn215974] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50790] 2015-10-29T01:21:24.043Z I NETWORK [conn205471] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:51968] 2015-10-29T01:21:24.043Z I NETWORK [conn205589] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37244] 2015-10-29T01:21:24.043Z I NETWORK [conn205943] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52189] 2015-10-29T01:21:24.058Z I NETWORK [conn206317] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35377] 2015-10-29T01:21:24.044Z I NETWORK [conn212600] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:37380] 2015-10-29T01:21:24.044Z I NETWORK [conn205609] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55284] 2015-10-29T01:21:24.059Z I NETWORK [conn205683] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52137] 2015-10-29T01:21:24.044Z I NETWORK [conn216071] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53562] 2015-10-29T01:21:24.044Z I NETWORK [conn205547] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55268] 2015-10-29T01:21:24.044Z I NETWORK [conn215926] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50742] 2015-10-29T01:21:24.034Z I NETWORK [conn205742] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37270] 2015-10-29T01:21:24.044Z I NETWORK [conn205577] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52125] 2015-10-29T01:21:24.059Z I NETWORK [conn205877] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37289] 2015-10-29T01:21:24.045Z I NETWORK [conn205528] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52120] 2015-10-29T01:21:24.045Z I NETWORK [conn183095] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:55458] 2015-10-29T01:21:24.045Z I NETWORK [conn205736] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37268] 2015-10-29T01:21:24.026Z I NETWORK [conn205828] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55325] 2015-10-29T01:21:24.045Z I NETWORK [conn215876] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50715] 2015-10-29T01:21:24.045Z I NETWORK [conn205868] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55336] 2015-10-29T01:21:24.045Z I NETWORK [conn207408] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45153] 2015-10-29T01:21:24.045Z I NETWORK [conn215858] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50704] 2015-10-29T01:21:24.060Z I NETWORK [conn205691] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52138] 2015-10-29T01:21:24.045Z I NETWORK [conn216063] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53550] 2015-10-29T01:21:24.046Z I NETWORK [conn205937] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37302] 2015-10-29T01:21:24.046Z I NETWORK [conn205938] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37303] 2015-10-29T01:21:24.060Z I NETWORK [conn205573] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55276] 2015-10-29T01:21:24.046Z I NETWORK [conn205671] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52135] 2015-10-29T01:21:24.060Z I NETWORK [conn206332] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35432] 2015-10-29T01:21:24.046Z I NETWORK [conn216082] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53573] 2015-10-29T01:21:24.060Z I NETWORK [conn216143] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53664] 2015-10-29T01:21:24.046Z I NETWORK [conn215145] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:60734] 2015-10-29T01:21:24.046Z I NETWORK [conn216104] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53625] 2015-10-29T01:21:24.047Z I NETWORK [conn206794] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39474] 2015-10-29T01:21:24.061Z I NETWORK [conn206768] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39393] 2015-10-29T01:21:24.047Z I NETWORK [conn205848] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37283] 2015-10-29T01:21:24.061Z I NETWORK [conn183099] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:34498] 2015-10-29T01:21:24.047Z I NETWORK [conn206313] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35373] 2015-10-29T01:21:24.047Z I NETWORK [conn207405] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45150] 2015-10-29T01:21:24.047Z I NETWORK [conn205935] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37301] 2015-10-29T01:21:24.047Z I NETWORK [conn205681] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55299] 2015-10-29T01:21:24.061Z I NETWORK [conn216134] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53655] 2015-10-29T01:21:24.061Z I NETWORK [conn205804] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52154] 2015-10-29T01:21:24.061Z I NETWORK [conn205760] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37272] 2015-10-29T01:21:24.048Z I NETWORK [conn216064] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53551] 2015-10-29T01:21:24.048Z I NETWORK [conn205562] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55272] 2015-10-29T01:21:24.048Z I NETWORK [conn205554] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37234] 2015-10-29T01:21:24.048Z I NETWORK [conn206314] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35375] 2015-10-29T01:21:24.048Z I NETWORK [conn216152] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53673] 2015-10-29T01:21:24.062Z I NETWORK [conn205730] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52142] 2015-10-29T01:21:24.049Z I NETWORK [conn216096] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53617] 2015-10-29T01:21:24.062Z I NETWORK [conn215843] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50641] 2015-10-29T01:21:24.062Z I NETWORK [conn215921] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50737] 2015-10-29T01:21:24.049Z I NETWORK [conn205934] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52184] 2015-10-29T01:21:24.049Z I NETWORK [conn177440] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:33446] 2015-10-29T01:21:24.063Z I NETWORK [conn215927] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50743] 2015-10-29T01:21:24.063Z I NETWORK [conn215930] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50746] 2015-10-29T01:21:24.049Z I NETWORK [conn216149] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53670] 2015-10-29T01:21:24.049Z I NETWORK [conn216140] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53661] 2015-10-29T01:21:24.063Z I NETWORK [conn216113] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53634] 2015-10-29T01:21:24.050Z I NETWORK [conn216066] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53556] 2015-10-29T01:21:24.063Z I NETWORK [conn205422] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55045] 2015-10-29T01:21:24.050Z I NETWORK [conn170487] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:47323] 2015-10-29T01:21:24.063Z I NETWORK [conn216153] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53674] 2015-10-29T01:21:24.063Z I NETWORK [conn207417] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45162] 2015-10-29T01:21:24.051Z I NETWORK [conn216078] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53569] 2015-10-29T01:21:24.051Z I NETWORK [conn177808] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:48497] 2015-10-29T01:21:24.064Z I NETWORK [conn205852] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55330] 2015-10-29T01:21:24.051Z I NETWORK [conn215909] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50725] 2015-10-29T01:21:24.064Z I NETWORK [conn205579] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37240] 2015-10-29T01:21:24.038Z I NETWORK [conn206315] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35374] 2015-10-29T01:21:24.064Z I NETWORK [conn216160] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53681] 2015-10-29T01:21:24.051Z I NETWORK [conn205594] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37245] 2015-10-29T01:21:24.051Z I NETWORK [conn205678] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55297] 2015-10-29T01:21:24.052Z I NETWORK [conn216083] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53577] 2015-10-29T01:21:24.052Z I NETWORK [conn205737] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55302] 2015-10-29T01:21:24.052Z I NETWORK [conn205499] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55162] 2015-10-29T01:21:24.052Z I NETWORK [conn215978] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50794] 2015-10-29T01:21:24.064Z I NETWORK [conn205859] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55333] 2015-10-29T01:21:24.064Z I NETWORK [conn205784] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52151] 2015-10-29T01:21:24.052Z I NETWORK [conn183346] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:48077] 2015-10-29T01:21:24.064Z I NETWORK [conn216105] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53626] 2015-10-29T01:21:24.052Z I NETWORK [conn183342] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:48070] 2015-10-29T01:21:24.052Z I NETWORK [conn205498] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:51973] 2015-10-29T01:21:24.053Z I NETWORK [conn207413] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45158] 2015-10-29T01:21:24.065Z I NETWORK [conn206813] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39564] 2015-10-29T01:21:24.065Z I NETWORK [conn205639] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37250] 2015-10-29T01:21:24.039Z I NETWORK [conn207401] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45145] 2015-10-29T01:21:24.053Z I NETWORK [conn205587] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55280] 2015-10-29T01:21:24.065Z I NETWORK [conn215935] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50751] 2015-10-29T01:21:24.065Z I NETWORK [conn216051] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53430] 2015-10-29T01:21:24.065Z I NETWORK [conn205497] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55161] 2015-10-29T01:21:24.053Z I NETWORK [conn205933] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37300] 2015-10-29T01:21:24.039Z I NETWORK [conn205478] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37128] 2015-10-29T01:21:24.066Z I NETWORK [conn205690] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37257] 2015-10-29T01:21:24.054Z I NETWORK [conn215979] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50795] 2015-10-29T01:21:24.054Z I NETWORK [conn216118] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53639] 2015-10-29T01:21:24.054Z I NETWORK [conn205844] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52163] 2015-10-29T01:21:24.066Z I NETWORK [conn205799] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52153] 2015-10-29T01:21:24.066Z I NETWORK [conn205625] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55288] 2015-10-29T01:21:24.054Z I NETWORK [conn205526] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52118] 2015-10-29T01:21:24.054Z I NETWORK [conn205899] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55342] 2015-10-29T01:21:24.066Z I NETWORK [conn205613] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55285] 2015-10-29T01:21:24.054Z I NETWORK [conn205879] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52167] 2015-10-29T01:21:24.055Z I NETWORK [conn205866] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55335] 2015-10-29T01:21:24.055Z I NETWORK [conn215951] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50767] 2015-10-29T01:21:24.066Z I NETWORK [conn216092] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53613] 2015-10-29T01:21:24.067Z I NETWORK [conn215853] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50696] 2015-10-29T01:21:24.040Z I NETWORK [conn215134] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:60722] 2015-10-29T01:21:24.055Z I NETWORK [conn216060] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53547] 2015-10-29T01:21:24.055Z I NETWORK [conn205930] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37298] 2015-10-29T01:21:24.056Z I NETWORK [conn215962] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50778] 2015-10-29T01:21:24.056Z I NETWORK [conn205798] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37278] 2015-10-29T01:21:24.056Z I NETWORK [conn205516] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52113] 2015-10-29T01:21:24.056Z I NETWORK [conn177469] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:54687] 2015-10-29T01:21:24.041Z I NETWORK [conn205707] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37261] 2015-10-29T01:21:24.057Z I NETWORK [conn206261] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:34974] 2015-10-29T01:21:24.057Z I NETWORK [conn205839] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52162] 2015-10-29T01:21:24.057Z I NETWORK [conn205618] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55286] 2015-10-29T01:21:24.057Z I NETWORK [conn205831] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52161] 2015-10-29T01:21:24.057Z I NETWORK [conn205808] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55318] 2015-10-29T01:21:24.067Z I NETWORK [conn212596] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:37376] 2015-10-29T01:21:24.042Z I NETWORK [conn216098] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53619] 2015-10-29T01:21:24.057Z I NETWORK [conn205776] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52148] 2015-10-29T01:21:24.057Z I NETWORK [conn205607] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37246] 2015-10-29T01:21:24.057Z I NETWORK [conn205512] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52111] 2015-10-29T01:21:24.042Z I NETWORK [conn205630] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55290] 2015-10-29T01:21:24.067Z I NETWORK [conn206330] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35430] 2015-10-29T01:21:24.058Z I NETWORK [conn212228] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:33874] 2015-10-29T01:21:24.068Z I NETWORK [conn215148] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:60738] 2015-10-29T01:21:24.043Z I NETWORK [conn205915] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52171] 2015-10-29T01:21:24.068Z I NETWORK [conn216141] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53662] 2015-10-29T01:21:24.058Z I NETWORK [conn205454] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37122] 2015-10-29T01:21:24.058Z I NETWORK [conn215939] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50755] 2015-10-29T01:21:24.058Z I NETWORK [conn205473] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:51969] 2015-10-29T01:21:24.058Z I NETWORK [conn205880] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52168] 2015-10-29T01:21:24.059Z I NETWORK [conn206255] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:34965] 2015-10-29T01:21:24.059Z I NETWORK [conn216114] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53635] 2015-10-29T01:21:24.068Z I NETWORK [conn205481] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37129] 2015-10-29T01:21:24.059Z I NETWORK [conn205803] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55317] 2015-10-29T01:21:24.059Z I NETWORK [conn205810] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37279] 2015-10-29T01:21:24.044Z I NETWORK [conn215929] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50745] 2015-10-29T01:21:24.059Z I NETWORK [conn216080] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53571] 2015-10-29T01:21:24.059Z I NETWORK [conn205484] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55155] 2015-10-29T01:21:24.059Z I NETWORK [conn215963] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50779] 2015-10-29T01:21:24.059Z I NETWORK [conn215908] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50724] 2015-10-29T01:21:24.059Z I NETWORK [conn205840] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55327] 2015-10-29T01:21:24.059Z I NETWORK [conn183088] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:45692] 2015-10-29T01:21:24.059Z I NETWORK [conn216159] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53680] 2015-10-29T01:21:24.069Z I NETWORK [conn205599] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52128] 2015-10-29T01:21:24.060Z I NETWORK [conn216121] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53642] 2015-10-29T01:21:24.045Z I NETWORK [conn215943] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50759] 2015-10-29T01:21:24.060Z I NETWORK [conn206294] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35246] 2015-10-29T01:21:24.060Z I NETWORK [conn216068] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53558] 2015-10-29T01:21:24.069Z I NETWORK [conn206258] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:34971] 2015-10-29T01:21:24.060Z I NETWORK [conn205595] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55281] 2015-10-29T01:21:24.060Z I NETWORK [conn205778] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52149] 2015-10-29T01:21:24.046Z I NETWORK [conn216059] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53546] 2015-10-29T01:21:24.060Z I NETWORK [conn207416] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45161] 2015-10-29T01:21:24.061Z I NETWORK [conn170777] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:50020] 2015-10-29T01:21:24.047Z I NETWORK [conn206800] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39480] 2015-10-29T01:21:24.069Z I NETWORK [conn206771] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39396] 2015-10-29T01:21:24.061Z I NETWORK [conn205775] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37275] 2015-10-29T01:21:24.061Z I NETWORK [conn205931] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52179] 2015-10-29T01:21:24.069Z I NETWORK [conn215917] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50733] 2015-10-29T01:21:24.069Z I NETWORK [conn205500] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55163] 2015-10-29T01:21:24.061Z I NETWORK [conn212227] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:33873] 2015-10-29T01:21:24.069Z I NETWORK [conn216101] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53622] 2015-10-29T01:21:24.061Z I NETWORK [conn179981] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52187] 2015-10-29T01:21:24.027Z I NETWORK [conn177442] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:33448] 2015-10-29T01:21:24.048Z I NETWORK [conn205916] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52172] 2015-10-29T01:21:24.070Z I NETWORK [conn215143] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:60733] 2015-10-29T01:21:24.070Z I NETWORK [conn215972] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50788] 2015-10-29T01:21:24.062Z I NETWORK [conn206304] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35323] 2015-10-29T01:21:24.062Z I NETWORK [conn215941] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50757] 2015-10-29T01:21:24.062Z I NETWORK [conn215922] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50738] 2015-10-29T01:21:24.070Z I NETWORK [conn215957] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50773] 2015-10-29T01:21:24.062Z I NETWORK [conn205604] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52129] 2015-10-29T01:21:24.062Z I NETWORK [conn205944] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37306] 2015-10-29T01:21:24.062Z I NETWORK [conn206320] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35380] 2015-10-29T01:21:24.062Z I NETWORK [conn215983] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50799] 2015-10-29T01:21:24.070Z I NETWORK [conn206807] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39558] 2015-10-29T01:21:24.062Z I NETWORK [conn205818] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52159] 2015-10-29T01:21:24.038Z I NETWORK [conn205863] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37285] 2015-10-29T01:21:24.063Z I NETWORK [conn205585] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37242] 2015-10-29T01:21:24.063Z I NETWORK [conn205858] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37284] 2015-10-29T01:21:24.071Z I NETWORK [conn216076] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53567] 2015-10-29T01:21:24.071Z I NETWORK [conn205865] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55334] 2015-10-29T01:21:24.063Z I NETWORK [conn206796] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39476] 2015-10-29T01:21:24.063Z I NETWORK [conn215875] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50714] 2015-10-29T01:21:24.063Z I NETWORK [conn216124] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53645] 2015-10-29T01:21:24.050Z I NETWORK [conn215944] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50760] 2015-10-29T01:21:24.063Z I NETWORK [conn216074] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53565] 2015-10-29T01:21:24.063Z I NETWORK [conn205424] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55050] 2015-10-29T01:21:24.063Z I NETWORK [conn170473] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:58504] 2015-10-29T01:21:24.063Z I NETWORK [conn215914] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50730] 2015-10-29T01:21:24.051Z I NETWORK [conn216065] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53552] 2015-10-29T01:21:24.064Z I NETWORK [conn205788] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55314] 2015-10-29T01:21:24.071Z I NETWORK [conn205882] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55340] 2015-10-29T01:21:24.072Z I NETWORK [conn205455] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37123] 2015-10-29T01:21:24.064Z I NETWORK [conn215136] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:60725] 2015-10-29T01:21:24.064Z I NETWORK [conn205628] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55289] 2015-10-29T01:21:24.064Z I NETWORK [conn205740] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37269] 2015-10-29T01:21:24.072Z I NETWORK [conn215982] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50798] 2015-10-29T01:21:24.064Z I NETWORK [conn205908] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55345] 2015-10-29T01:21:24.052Z I NETWORK [conn216072] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53563] 2015-10-29T01:21:24.064Z I NETWORK [conn215918] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50734] 2015-10-29T01:21:24.065Z I NETWORK [conn216070] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53561] 2015-10-29T01:21:24.072Z I NETWORK [conn205644] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37251] 2015-10-29T01:21:24.065Z I NETWORK [conn215971] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50787] 2015-10-29T01:21:24.053Z I NETWORK [conn205482] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37130] 2015-10-29T01:21:24.065Z I NETWORK [conn207473] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:45454] 2015-10-29T01:21:24.072Z I NETWORK [conn216073] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53564] 2015-10-29T01:21:24.053Z I NETWORK [conn205634] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55291] 2015-10-29T01:21:24.065Z I NETWORK [conn205768] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55307] 2015-10-29T01:21:24.065Z I NETWORK [conn177379] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:44211] 2015-10-29T01:21:24.065Z I NETWORK [conn205917] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52173] 2015-10-29T01:21:24.066Z I NETWORK [conn216061] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53548] 2015-10-29T01:21:24.066Z I NETWORK [conn215968] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50784] 2015-10-29T01:21:24.066Z I NETWORK [conn205561] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37235] 2015-10-29T01:21:24.073Z I NETWORK [conn205801] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55316] 2015-10-29T01:21:24.055Z I NETWORK [conn205621] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55287] 2015-10-29T01:21:24.066Z I NETWORK [conn206257] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:34967] 2015-10-29T01:21:24.066Z I NETWORK [conn183114] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:34604] 2015-10-29T01:21:24.067Z I NETWORK [conn206312] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35372] 2015-10-29T01:21:24.073Z I NETWORK [conn205496] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55160] 2015-10-29T01:21:24.056Z I NETWORK [conn205470] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37127] 2015-10-29T01:21:24.073Z I NETWORK [conn216158] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53679] 2015-10-29T01:21:24.067Z I NETWORK [conn205442] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37119] 2015-10-29T01:21:24.067Z I NETWORK [conn206331] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35431] 2015-10-29T01:21:24.067Z I NETWORK [conn205584] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37241] 2015-10-29T01:21:24.067Z I NETWORK [conn205928] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52178] 2015-10-29T01:21:24.057Z I NETWORK [conn215138] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:60727] 2015-10-29T01:21:24.068Z I NETWORK [conn205728] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52141] 2015-10-29T01:21:24.073Z I NETWORK [conn215149] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:60739] 2015-10-29T01:21:24.073Z I NETWORK [conn170504] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:40286] 2015-10-29T01:21:24.069Z I NETWORK [conn216103] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53624] 2015-10-29T01:21:24.073Z I NETWORK [conn216122] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53643] 2015-10-29T01:21:24.069Z I NETWORK [conn216069] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53559] 2015-10-29T01:21:24.069Z I NETWORK [conn215952] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50768] 2015-10-29T01:21:24.061Z I NETWORK [conn206305] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35322] 2015-10-29T01:21:24.074Z I NETWORK [conn215949] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50765] 2015-10-29T01:21:24.074Z I NETWORK [conn205550] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37233] 2015-10-29T01:21:24.070Z I NETWORK [conn206319] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35379] 2015-10-29T01:21:24.062Z I NETWORK [conn216136] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53657] 2015-10-29T01:21:24.062Z I NETWORK [conn177807] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:48498] 2015-10-29T01:21:24.071Z I NETWORK [conn205568] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55273] 2015-10-29T01:21:24.063Z I NETWORK [conn216119] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53640] 2015-10-29T01:21:24.074Z I NETWORK [conn216144] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53665] 2015-10-29T01:21:24.063Z I NETWORK [conn205708] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37262] 2015-10-29T01:21:24.038Z I NETWORK [conn205522] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52117] 2015-10-29T01:21:24.071Z I NETWORK [conn216126] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53647] 2015-10-29T01:21:24.071Z I NETWORK [conn205629] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37248] 2015-10-29T01:21:24.071Z I NETWORK [conn206810] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39561] 2015-10-29T01:21:24.064Z I NETWORK [conn205946] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52191] 2015-10-29T01:21:24.072Z I NETWORK [conn215924] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50740] 2015-10-29T01:21:24.072Z I NETWORK [conn215850] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50693] 2015-10-29T01:21:24.065Z I NETWORK [conn206776] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39401] 2015-10-29T01:21:24.072Z I NETWORK [conn170472] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:58501] 2015-10-29T01:21:24.073Z I NETWORK [conn215953] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50769] 2015-10-29T01:21:24.066Z I NETWORK [conn215857] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50703] 2015-10-29T01:21:24.073Z I NETWORK [conn216090] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53611] 2015-10-29T01:21:24.073Z I NETWORK [conn206311] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:35371] 2015-10-29T01:21:24.073Z I NETWORK [conn215878] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50717] 2015-10-29T01:21:24.067Z I NETWORK [conn216094] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53615] 2015-10-29T01:21:24.056Z I NETWORK [conn215859] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50705] 2015-10-29T01:21:24.073Z I NETWORK [conn215910] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.245:50726] 2015-10-29T01:21:24.068Z I NETWORK [conn205571] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55274] 2015-10-29T01:21:24.060Z I NETWORK [conn205843] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55329] 2015-10-29T01:21:24.060Z I NETWORK [conn206798] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.195:39478] 2015-10-29T01:21:24.061Z I NETWORK [conn205940] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:52188] 2015-10-29T01:21:24.074Z I NETWORK [conn216111] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53632] 2015-10-29T01:21:24.074Z I NETWORK [conn183104] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.2.110:55509] 2015-10-29T01:21:24.075Z I NETWORK [conn216112] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.0.144:53633] 2015-10-29T01:21:24.074Z I NETWORK [conn205559] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.25:55271] 2015-10-29T01:21:24.074Z I NETWORK [conn205881] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.1.224:37291] 2015-10-29T01:21:24.983Z I REPL [ReplicationExecutor] replSetElect voting yea for mongodb-upe_dccb-3.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 (6) 2015-10-29T01:21:25.110Z I REPL [ReplicationExecutor] Member mongodb-upe_dccb-3.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 is now in state PRIMARY 2015-10-29T01:21:27.513Z I REPL [ReplicationExecutor] syncing from: mongodb-upe_dccb-3.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 2015-10-29T01:21:27.803Z I REPL [SyncSourceFeedback] replset setting syncSourceFeedback to mongodb-upe_dccb-3.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 2015-10-29T01:21:27.805Z I NETWORK [conn225388] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.79:42374] 2015-10-29T01:21:27.805Z I NETWORK [conn225387] SocketException handling request, closing client connection: 9001 socket exception [SEND_ERROR] server [172.19.101.157:42281] 2015-10-29T01:21:42.841Z I - [conn225391] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:21:42.848Z I CONTROL [conn225391] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:21:42.849Z I QUERY [conn225391] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-29T01:21:44.607Z I - [conn225391] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:21:44.615Z I CONTROL [conn225391] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:21:44.615Z I QUERY [conn225391] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-29T01:21:45.869Z I - [conn225391] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:21:45.877Z I CONTROL [conn225391] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:21:45.877Z I QUERY [conn225391] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-29T01:21:47.009Z I - [conn225391] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:21:47.017Z I CONTROL [conn225391] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:21:47.017Z I QUERY [conn225391] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-29T01:22:12.603Z I - [conn225391] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:22:12.611Z I CONTROL [conn225391] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:22:12.611Z I QUERY [conn225391] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-29T01:25:20.201Z I - [conn226011] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T01:25:20.209Z I CONTROL [conn226011] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FDE2D03E000","o":"7DF5"},{"b":"7FDE2BA02000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFF5D5FC000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FDE2D03E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FDE2CDD1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FDE2C9EC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FDE2C7E4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FDE2C5E0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FDE2C2DC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FDE2BFDA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FDE2BDC4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FDE2BA02000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FDE2D25A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FDE2B7B6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FDE2B4D3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FDE2B2D0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FDE2B09E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FDE2AE88000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FDE2AC79000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FDE2AA76000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FDE2A85C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FDE2A63B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fde2d045df5] libc.so.6(clone+0x6D) [0x7fde2baf8bfd] ----- END BACKTRACE ----- 2015-10-29T01:25:20.209Z I QUERY [conn226011] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-29T01:33:07.609Z I REPL [conn226011] Stopping replication applier threads 2015-10-29T01:33:07.701Z I CONTROL [conn226011] now exiting 2015-10-29T01:33:07.701Z I NETWORK [conn226011] shutdown: going to close listening sockets... 2015-10-29T01:33:07.701Z I NETWORK [conn226011] closing listening socket: 6 2015-10-29T01:33:07.701Z I NETWORK [conn226011] closing listening socket: 7 2015-10-29T01:33:07.701Z I NETWORK [conn226011] removing socket file: /tmp/mongodb-27017.sock 2015-10-29T01:33:07.701Z I NETWORK [conn226011] shutdown: going to flush diaglog... 2015-10-29T01:33:07.701Z I NETWORK [conn226011] shutdown: going to close sockets... 2015-10-29T01:33:11.259Z I CONTROL [conn226011] dbexit: rc: 0 2015-10-29T01:34:48.671Z I CONTROL ***** SERVER RESTARTED ***** 2015-10-29T01:34:48.756Z I CONTROL [initandlisten] MongoDB starting : pid=43739 port=27017 dbpath=/mongodata/db 64-bit host=ip-172-19-100-23 2015-10-29T01:34:48.756Z I CONTROL [initandlisten] db version v3.0.5 2015-10-29T01:34:48.756Z I CONTROL [initandlisten] git version: 8bc4ae20708dbb493cb09338d9e7be6698e4a3a3 2015-10-29T01:34:48.756Z I CONTROL [initandlisten] build info: Linux ip-10-230-137-3 3.4.43-43.43.amzn1.x86_64 #1 SMP Mon May 6 18:04:41 UTC 2013 x86_64 BOOST_LIB_VERSION=1_49 2015-10-29T01:34:48.756Z I CONTROL [initandlisten] allocator: tcmalloc 2015-10-29T01:34:48.756Z I CONTROL [initandlisten] options: { config: "/etc/mongod.conf", net: { port: 27017, ssl: { PEMKeyFile: "/mongodata/config/mongodb.pem", PEMKeyPassword: "", mode: "preferSSL" } }, operationProfiling: { mode: "off", slowOpThresholdMs: 100 }, processManagement: { fork: true, pidFilePath: "/var/run/mongodb/mongod.pid" }, replication: { oplogSizeMB: 20480, replSetName: "upe_dccb" }, security: { clusterAuthMode: "keyFile", keyFile: "/mongodata/config/keyfile" }, storage: { dbPath: "/mongodata/db", directoryPerDB: true, engine: "wiredTiger", wiredTiger: { collectionConfig: { blockCompressor: "snappy" }, engineConfig: { directoryForIndexes: true, journalCompressor: "snappy", statisticsLogDelaySecs: 0 }, indexConfig: { prefixCompression: true } } }, systemLog: { destination: "file", logAppend: true, logRotate: "reopen", path: "/data/log/mongodb/mongodb.log", timeStampFormat: "iso8601-utc", traceAllExceptions: false, verbosity: 0 } } 2015-10-29T01:34:48.767Z I REPL [initandlisten] Did not find local replica set configuration document at startup; NoMatchingDocument Did not find replica set configuration document in local.system.replset 2015-10-29T01:34:48.776Z I NETWORK [initandlisten] waiting for connections on port 27017 ssl 2015-10-29T01:34:50.220Z I REPL [WriteReplSetConfig] Starting replication applier threads 2015-10-29T01:34:50.220Z I REPL [rsSync] replSet warning did not receive a valid config yet, sleeping 5 seconds 2015-10-29T01:34:50.221Z I REPL [ReplicationExecutor] New replica set config in use: { _id: "upe_dccb", version: 11496652, members: [ { _id: 4, host: "mongodb-upe_dccb-1.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017", arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 5, host: "mongodb-upe_dccb-2.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017", arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 6, host: "mongodb-upe_dccb-3.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017", arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 } ], settings: { chainingAllowed: true, heartbeatTimeoutSecs: 10, getLastErrorModes: {}, getLastErrorDefaults: { w: 1, wtimeout: 0 } } } 2015-10-29T01:34:50.221Z I REPL [ReplicationExecutor] This node is mongodb-upe_dccb-1.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 in the config 2015-10-29T01:34:50.221Z I REPL [ReplicationExecutor] transition to STARTUP2 2015-10-29T01:34:50.221Z I REPL [ReplicationExecutor] Member mongodb-upe_dccb-3.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 is now in state PRIMARY 2015-10-29T01:34:50.221Z I REPL [ReplicationExecutor] Member mongodb-upe_dccb-2.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 is now in state SECONDARY 2015-10-29T01:34:55.221Z I REPL [rsSync] ****** 2015-10-29T01:34:55.221Z I REPL [rsSync] creating replication oplog of size: 20480MB... 2015-10-29T01:34:55.265Z I REPL [rsSync] ****** 2015-10-29T01:34:55.265Z I REPL [rsSync] initial sync pending 2015-10-29T01:34:55.274Z I REPL [ReplicationExecutor] syncing from: mongodb-upe_dccb-3.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 2015-10-29T01:34:55.569Z I REPL [rsSync] initial sync drop all databases 2015-10-29T01:34:55.569Z I REPL [rsSync] initial sync clone all databases 2015-10-29T01:34:55.585Z I REPL [rsSync] initial sync cloning db: admin 2015-10-29T01:34:55.656Z I INDEX [rsSync] build index on: admin.system.users properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "admin.system.users" } 2015-10-29T01:34:55.656Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:55.660Z I INDEX [rsSync] build index done. scanned 5 total records. 0 secs 2015-10-29T01:34:55.670Z I INDEX [rsSync] build index on: admin.system.version properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "admin.system.version" } 2015-10-29T01:34:55.670Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:55.673Z I INDEX [rsSync] build index done. scanned 1 total records. 0 secs 2015-10-29T01:34:55.674Z I REPL [rsSync] initial sync cloning db: configuration 2015-10-29T01:34:55.716Z I INDEX [rsSync] build index on: configuration.config_entries properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "configuration.config_entries" } 2015-10-29T01:34:55.716Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:55.720Z I INDEX [rsSync] build index done. scanned 265 total records. 0 secs 2015-10-29T01:34:55.730Z I INDEX [rsSync] build index on: configuration.health_check properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "configuration.health_check" } 2015-10-29T01:34:55.730Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:55.734Z I INDEX [rsSync] build index done. scanned 5 total records. 0 secs 2015-10-29T01:34:55.734Z I REPL [rsSync] initial sync cloning db: endpoint_mcs 2015-10-29T01:34:55.777Z I INDEX [rsSync] build index on: endpoint_mcs.customers properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "endpoint_mcs.customers" } 2015-10-29T01:34:55.777Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:55.783Z I INDEX [rsSync] build index done. scanned 268 total records. 0 secs 2015-10-29T01:34:55.793Z I INDEX [rsSync] build index on: endpoint_mcs.data_migrations properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "endpoint_mcs.data_migrations" } 2015-10-29T01:34:55.793Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:55.798Z I INDEX [rsSync] build index done. scanned 3 total records. 0 secs 2015-10-29T01:34:55.810Z I INDEX [rsSync] build index on: endpoint_mcs.endpoint_deployments properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "endpoint_mcs.endpoint_deployments" } 2015-10-29T01:34:55.810Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:55.817Z I INDEX [rsSync] build index done. scanned 236 total records. 0 secs 2015-10-29T01:34:55.829Z I INDEX [rsSync] build index on: endpoint_mcs.endpoint_installers properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "endpoint_mcs.endpoint_installers" } 2015-10-29T01:34:55.829Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:55.836Z I INDEX [rsSync] build index done. scanned 472 total records. 0 secs 2015-10-29T01:34:55.904Z I INDEX [rsSync] build index on: endpoint_mcs.endpoints properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "endpoint_mcs.endpoints" } 2015-10-29T01:34:55.905Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:55.925Z I INDEX [rsSync] build index done. scanned 2385 total records. 0 secs 2015-10-29T01:34:55.934Z I INDEX [rsSync] build index on: endpoint_mcs.ham_mq_envelopes properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "endpoint_mcs.ham_mq_envelopes" } 2015-10-29T01:34:55.934Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:55.938Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:34:55.947Z I INDEX [rsSync] build index on: endpoint_mcs.ham_mq_group_locks properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "endpoint_mcs.ham_mq_group_locks" } 2015-10-29T01:34:55.947Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:55.949Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:34:55.959Z I INDEX [rsSync] build index on: endpoint_mcs.health_check properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "endpoint_mcs.health_check" } 2015-10-29T01:34:55.959Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:55.963Z I INDEX [rsSync] build index done. scanned 3 total records. 0 secs 2015-10-29T01:34:55.963Z I REPL [rsSync] initial sync cloning db: htest 2015-10-29T01:34:56.006Z I INDEX [rsSync] build index on: htest.endpoints properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "htest.endpoints" } 2015-10-29T01:34:56.006Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.010Z I INDEX [rsSync] build index done. scanned 1 total records. 0 secs 2015-10-29T01:34:56.010Z I REPL [rsSync] initial sync cloning db: mobile 2015-10-29T01:34:56.051Z I INDEX [rsSync] build index on: mobile.app_data properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "mobile.app_data" } 2015-10-29T01:34:56.051Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.055Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:34:56.065Z I INDEX [rsSync] build index on: mobile.customers properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "mobile.customers" } 2015-10-29T01:34:56.065Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.068Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:34:56.078Z I INDEX [rsSync] build index on: mobile.data_migrations properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "mobile.data_migrations" } 2015-10-29T01:34:56.078Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.081Z I INDEX [rsSync] build index done. scanned 2 total records. 0 secs 2015-10-29T01:34:56.091Z I INDEX [rsSync] build index on: mobile.device_actions properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "mobile.device_actions" } 2015-10-29T01:34:56.091Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.094Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:34:56.163Z I INDEX [rsSync] build index on: mobile.device_models properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "mobile.device_models" } 2015-10-29T01:34:56.163Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.184Z I INDEX [rsSync] build index done. scanned 7912 total records. 0 secs 2015-10-29T01:34:56.194Z I INDEX [rsSync] build index on: mobile.devices properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "mobile.devices" } 2015-10-29T01:34:56.194Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.196Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:34:56.205Z I INDEX [rsSync] build index on: mobile.enrollments properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "mobile.enrollments" } 2015-10-29T01:34:56.205Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.208Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:34:56.217Z I INDEX [rsSync] build index on: mobile.ham_mq_envelopes properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "mobile.ham_mq_envelopes" } 2015-10-29T01:34:56.217Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.219Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:34:56.228Z I INDEX [rsSync] build index on: mobile.ham_mq_group_locks properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "mobile.ham_mq_group_locks" } 2015-10-29T01:34:56.228Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.231Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:34:56.240Z I INDEX [rsSync] build index on: mobile.health_check properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "mobile.health_check" } 2015-10-29T01:34:56.240Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.243Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:34:56.243Z I REPL [rsSync] initial sync cloning db: mobile_dev 2015-10-29T01:34:56.285Z I INDEX [rsSync] build index on: mobile_dev.device_models properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "mobile_dev.device_models" } 2015-10-29T01:34:56.285Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.287Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:34:56.296Z I INDEX [rsSync] build index on: mobile_dev.devices properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "mobile_dev.devices" } 2015-10-29T01:34:56.296Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.299Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:34:56.308Z I INDEX [rsSync] build index on: mobile_dev.enrollments properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "mobile_dev.enrollments" } 2015-10-29T01:34:56.308Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.311Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:34:56.320Z I INDEX [rsSync] build index on: mobile_dev.health_check properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "mobile_dev.health_check" } 2015-10-29T01:34:56.320Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.322Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:34:56.322Z I REPL [rsSync] initial sync cloning db: scalability 2015-10-29T01:34:56.384Z I INDEX [rsSync] build index on: scalability.MetaCustomers properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "scalability.MetaCustomers" } 2015-10-29T01:34:56.384Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:56.395Z I INDEX [rsSync] build index done. scanned 2405 total records. 0 secs 2015-10-29T01:34:57.925Z I INDEX [rsSync] build index on: scalability.MetaEndpoints properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "scalability.MetaEndpoints" } 2015-10-29T01:34:57.925Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:58.554Z I INDEX [rsSync] build index done. scanned 240500 total records. 0 secs 2015-10-29T01:34:58.573Z I INDEX [rsSync] build index on: scalability.customer_groups properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "scalability.customer_groups" } 2015-10-29T01:34:58.573Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:58.577Z I INDEX [rsSync] build index done. scanned 29 total records. 0 secs 2015-10-29T01:34:58.585Z I INDEX [rsSync] build index on: scalability.scalability_counters properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "scalability.scalability_counters" } 2015-10-29T01:34:58.585Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:58.588Z I INDEX [rsSync] build index done. scanned 1 total records. 0 secs 2015-10-29T01:34:58.588Z I REPL [rsSync] initial sync cloning db: sledge 2015-10-29T01:34:58.631Z I INDEX [rsSync] build index on: sledge.alerts properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.alerts" } 2015-10-29T01:34:58.631Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:58.634Z I INDEX [rsSync] build index done. scanned 9 total records. 0 secs 2015-10-29T01:34:58.652Z I INDEX [rsSync] build index on: sledge.business_intelligence_reports properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.business_intelligence_reports" } 2015-10-29T01:34:58.652Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:58.658Z I INDEX [rsSync] build index done. scanned 72 total records. 0 secs 2015-10-29T01:34:58.667Z I INDEX [rsSync] build index on: sledge.controlled_application_categories properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.controlled_application_categories" } 2015-10-29T01:34:58.667Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:58.671Z I INDEX [rsSync] build index done. scanned 56 total records. 0 secs 2015-10-29T01:34:58.687Z I INDEX [rsSync] build index on: sledge.controlled_applications properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.controlled_applications" } 2015-10-29T01:34:58.687Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:58.695Z I INDEX [rsSync] build index done. scanned 1514 total records. 0 secs 2015-10-29T01:34:58.976Z I INDEX [rsSync] build index on: sledge.customer_flags properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.customer_flags" } 2015-10-29T01:34:58.976Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:59.082Z I INDEX [rsSync] build index done. scanned 43290 total records. 0 secs 2015-10-29T01:34:59.188Z I INDEX [rsSync] build index on: sledge.customers properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.customers" } 2015-10-29T01:34:59.188Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:59.214Z I INDEX [rsSync] build index done. scanned 2405 total records. 0 secs 2015-10-29T01:34:59.251Z I INDEX [rsSync] build index on: sledge.deployment_tokens properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.deployment_tokens" } 2015-10-29T01:34:59.252Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:34:59.266Z I INDEX [rsSync] build index done. scanned 4387 total records. 0 secs 2015-10-29T01:35:01.560Z I INDEX [rsSync] build index on: sledge.directory_objects properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.directory_objects" } 2015-10-29T01:35:01.560Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:35:02.490Z I INDEX [rsSync] build index done. scanned 327067 total records. 0 secs 2015-10-29T01:35:04.971Z I INDEX [rsSync] build index on: sledge.directory_user_matches properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.directory_user_matches" } 2015-10-29T01:35:04.971Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:35:05.892Z I INDEX [rsSync] build index done. scanned 335957 total records. 0 secs 2015-10-29T01:35:05.912Z I INDEX [rsSync] build index on: sledge.endpoint_credentials properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.endpoint_credentials" } 2015-10-29T01:35:05.912Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:35:05.915Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:35:07.082Z I INDEX [rsSync] build index on: sledge.endpoint_devices properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.endpoint_devices" } 2015-10-29T01:35:07.082Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:35:07.569Z I INDEX [rsSync] build index done. scanned 147594 total records. 0 secs 2015-10-29T01:35:08.158Z I INDEX [rsSync] build index on: sledge.endpoint_threats properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.endpoint_threats" } 2015-10-29T01:35:08.158Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:35:08.380Z I INDEX [rsSync] build index done. scanned 76088 total records. 0 secs 2015-10-29T01:35:14.693Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.endpoints" } 2015-10-29T01:35:14.693Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:35:15.998Z I INDEX [rsSync] build index done. scanned 221260 total records. 1 secs 2015-10-29T01:36:01.955Z I INDEX [rsSync] build index on: sledge.event_bases properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.event_bases" } 2015-10-29T01:36:01.955Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:04.002Z I - [rsSync] Index Build: 1925200/5519129 34% 2015-10-29T01:36:08.385Z I - [rsSync] Index Build: 2016500/5519129 36% 2015-10-29T01:36:14.984Z I - [rsSync] Index Build: 4033000/5519129 73% 2015-10-29T01:36:24.547Z I INDEX [rsSync] build index done. scanned 5519129 total records. 22 secs 2015-10-29T01:36:24.609Z I INDEX [rsSync] build index on: sledge.global_exclusions properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.global_exclusions" } 2015-10-29T01:36:24.609Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:24.620Z I INDEX [rsSync] build index done. scanned 2405 total records. 0 secs 2015-10-29T01:36:24.629Z I INDEX [rsSync] build index on: sledge.health_check properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.health_check" } 2015-10-29T01:36:24.629Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:24.632Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:24.641Z I INDEX [rsSync] build index on: sledge.home_intelligence_reports properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.home_intelligence_reports" } 2015-10-29T01:36:24.641Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:24.645Z I INDEX [rsSync] build index done. scanned 62 total records. 0 secs 2015-10-29T01:36:24.654Z I INDEX [rsSync] build index on: sledge.local_sites properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.local_sites" } 2015-10-29T01:36:24.654Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:24.656Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:26.577Z I INDEX [rsSync] build index on: sledge.mcs_commands properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.mcs_commands" } 2015-10-29T01:36:26.577Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:27.244Z I INDEX [rsSync] build index done. scanned 192792 total records. 0 secs 2015-10-29T01:36:27.343Z I INDEX [rsSync] build index on: sledge.mcs_policy_bases properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.mcs_policy_bases" } 2015-10-29T01:36:27.343Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:27.375Z I INDEX [rsSync] build index done. scanned 9620 total records. 0 secs 2015-10-29T01:36:27.428Z I INDEX [rsSync] build index on: sledge.message_schedules properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.message_schedules" } 2015-10-29T01:36:27.428Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:27.432Z I INDEX [rsSync] build index done. scanned 14 total records. 0 secs 2015-10-29T01:36:27.442Z I INDEX [rsSync] build index on: sledge.messages properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.messages" } 2015-10-29T01:36:27.442Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:27.445Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:27.796Z I INDEX [rsSync] build index on: sledge.policies properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.policies" } 2015-10-29T01:36:27.796Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:27.900Z I INDEX [rsSync] build index done. scanned 15490 total records. 0 secs 2015-10-29T01:36:41.575Z I INDEX [rsSync] build index on: sledge.policy_assignments properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.policy_assignments" } 2015-10-29T01:36:41.575Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:46.568Z I INDEX [rsSync] build index done. scanned 2202980 total records. 4 secs 2015-10-29T01:36:47.012Z I INDEX [rsSync] build index on: sledge.policy_fragments properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.policy_fragments" } 2015-10-29T01:36:47.012Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:47.148Z I INDEX [rsSync] build index done. scanned 22593 total records. 0 secs 2015-10-29T01:36:47.165Z I INDEX [rsSync] build index on: sledge.propertys properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.propertys" } 2015-10-29T01:36:47.165Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:47.169Z I INDEX [rsSync] build index done. scanned 2 total records. 0 secs 2015-10-29T01:36:47.385Z I INDEX [rsSync] build index on: sledge.rendered_policies properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.rendered_policies" } 2015-10-29T01:36:47.385Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:47.492Z I INDEX [rsSync] build index done. scanned 27395 total records. 0 secs 2015-10-29T01:36:49.529Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.report_bases" } 2015-10-29T01:36:49.529Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:50.203Z I INDEX [rsSync] build index done. scanned 292150 total records. 0 secs 2015-10-29T01:36:50.228Z I INDEX [rsSync] build index on: sledge.scheduled_messages properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.scheduled_messages" } 2015-10-29T01:36:50.228Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:50.230Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:51.262Z I INDEX [rsSync] build index on: sledge.sequences properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.sequences" } 2015-10-29T01:36:51.262Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:51.971Z I INDEX [rsSync] build index done. scanned 222740 total records. 0 secs 2015-10-29T01:36:52.021Z I INDEX [rsSync] build index on: sledge.sophos_articles properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.sophos_articles" } 2015-10-29T01:36:52.021Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.026Z I INDEX [rsSync] build index done. scanned 288 total records. 0 secs 2015-10-29T01:36:52.037Z I INDEX [rsSync] build index on: sledge.sophos_labs_threat_level properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.sophos_labs_threat_level" } 2015-10-29T01:36:52.037Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.040Z I INDEX [rsSync] build index done. scanned 1 total records. 0 secs 2015-10-29T01:36:52.097Z I INDEX [rsSync] build index on: sledge.sophos_labs_threat_volumes properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.sophos_labs_threat_volumes" } 2015-10-29T01:36:52.097Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.120Z I INDEX [rsSync] build index done. scanned 9894 total records. 0 secs 2015-10-29T01:36:52.153Z I INDEX [rsSync] build index on: sledge.sophos_labs_threats properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "sledge.sophos_labs_threats" } 2015-10-29T01:36:52.153Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.162Z I INDEX [rsSync] build index done. scanned 2749 total records. 0 secs 2015-10-29T01:36:52.163Z I REPL [rsSync] initial sync cloning db: test 2015-10-29T01:36:52.202Z I INDEX [rsSync] build index on: test.customer_flags properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "test.customer_flags" } 2015-10-29T01:36:52.202Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.206Z I INDEX [rsSync] build index done. scanned 54 total records. 0 secs 2015-10-29T01:36:52.215Z I INDEX [rsSync] build index on: test.customers properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "test.customers" } 2015-10-29T01:36:52.215Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.218Z I INDEX [rsSync] build index done. scanned 3 total records. 0 secs 2015-10-29T01:36:52.228Z I INDEX [rsSync] build index on: test.deployment_tokens properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "test.deployment_tokens" } 2015-10-29T01:36:52.228Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.231Z I INDEX [rsSync] build index done. scanned 3 total records. 0 secs 2015-10-29T01:36:52.242Z I INDEX [rsSync] build index on: test.directory_objects properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "test.directory_objects" } 2015-10-29T01:36:52.242Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.245Z I INDEX [rsSync] build index done. scanned 7 total records. 0 secs 2015-10-29T01:36:52.254Z I INDEX [rsSync] build index on: test.directory_user_matches properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "test.directory_user_matches" } 2015-10-29T01:36:52.254Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.260Z I INDEX [rsSync] build index done. scanned 3 total records. 0 secs 2015-10-29T01:36:52.269Z I INDEX [rsSync] build index on: test.endpoints properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "test.endpoints" } 2015-10-29T01:36:52.269Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.272Z I INDEX [rsSync] build index done. scanned 3 total records. 0 secs 2015-10-29T01:36:52.281Z I INDEX [rsSync] build index on: test.event_bases properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "test.event_bases" } 2015-10-29T01:36:52.281Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.285Z I INDEX [rsSync] build index done. scanned 3 total records. 0 secs 2015-10-29T01:36:52.294Z I INDEX [rsSync] build index on: test.global_exclusions properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "test.global_exclusions" } 2015-10-29T01:36:52.294Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.298Z I INDEX [rsSync] build index done. scanned 3 total records. 0 secs 2015-10-29T01:36:52.307Z I INDEX [rsSync] build index on: test.mcs_policy_bases properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "test.mcs_policy_bases" } 2015-10-29T01:36:52.307Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.313Z I INDEX [rsSync] build index done. scanned 12 total records. 0 secs 2015-10-29T01:36:52.325Z I INDEX [rsSync] build index on: test.policies properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "test.policies" } 2015-10-29T01:36:52.325Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.330Z I INDEX [rsSync] build index done. scanned 24 total records. 0 secs 2015-10-29T01:36:52.342Z I INDEX [rsSync] build index on: test.policy_assignments properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "test.policy_assignments" } 2015-10-29T01:36:52.342Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.346Z I INDEX [rsSync] build index done. scanned 30 total records. 0 secs 2015-10-29T01:36:52.356Z I INDEX [rsSync] build index on: test.policy_fragments properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "test.policy_fragments" } 2015-10-29T01:36:52.356Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.360Z I INDEX [rsSync] build index done. scanned 28 total records. 0 secs 2015-10-29T01:36:52.369Z I INDEX [rsSync] build index on: test.rendered_policies properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "test.rendered_policies" } 2015-10-29T01:36:52.369Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.372Z I INDEX [rsSync] build index done. scanned 23 total records. 0 secs 2015-10-29T01:36:52.372Z I REPL [rsSync] initial sync cloning db: upe_dev 2015-10-29T01:36:52.414Z I INDEX [rsSync] build index on: upe_dev.customers properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "upe_dev.customers" } 2015-10-29T01:36:52.414Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.418Z I INDEX [rsSync] build index done. scanned 2 total records. 0 secs 2015-10-29T01:36:52.418Z I REPL [rsSync] initial sync cloning db: zero_dev 2015-10-29T01:36:52.456Z I INDEX [rsSync] build index on: zero_dev.health_check properties: { v: 1, key: { _id: 1 }, name: "_id_", ns: "zero_dev.health_check" } 2015-10-29T01:36:52.456Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:52.458Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:52.458Z I REPL [rsSync] initial sync data copy, starting syncup 2015-10-29T01:36:52.459Z I REPL [rsSync] oplog sync 1 of 3 2015-10-29T01:36:53.237Z I REPL [ReplicationExecutor] syncing from: mongodb-upe_dccb-3.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 2015-10-29T01:36:53.599Z I REPL [SyncSourceFeedback] replset setting syncSourceFeedback to mongodb-upe_dccb-3.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 2015-10-29T01:36:55.160Z I REPL [rsSync] oplog sync 2 of 3 2015-10-29T01:36:55.181Z I REPL [rsSync] initial sync building indexes 2015-10-29T01:36:55.181Z I REPL [rsSync] initial sync cloning indexes for : admin 2015-10-29T01:36:55.214Z I INDEX [rsSync] build index on: admin.system.users properties: { v: 1, unique: true, key: { user: 1, db: 1 }, name: "user_1_db_1", ns: "admin.system.users" } 2015-10-29T01:36:55.214Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.242Z I INDEX [rsSync] build index done. scanned 5 total records. 0 secs 2015-10-29T01:36:55.243Z I REPL [rsSync] initial sync cloning indexes for : configuration 2015-10-29T01:36:55.275Z I INDEX [rsSync] build index on: configuration.config_entries properties: { v: 1, key: { context: 1 }, name: "context_1", ns: "configuration.config_entries", background: true } 2015-10-29T01:36:55.275Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.277Z I INDEX [rsSync] build index on: configuration.config_entries properties: { v: 1, key: { context: 1, key: 1 }, name: "context_1_key_1", ns: "configuration.config_entries", background: true } 2015-10-29T01:36:55.277Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.286Z I INDEX [rsSync] build index done. scanned 265 total records. 0 secs 2015-10-29T01:36:55.286Z I REPL [rsSync] initial sync cloning indexes for : endpoint_mcs 2015-10-29T01:36:55.320Z I INDEX [rsSync] build index on: endpoint_mcs.endpoint_deployments properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "endpoint_mcs.endpoint_deployments", sparse: false, background: false } 2015-10-29T01:36:55.320Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.322Z I INDEX [rsSync] build index on: endpoint_mcs.endpoint_deployments properties: { v: 1, key: { upstream_token: 1 }, name: "upstream_token_1", ns: "endpoint_mcs.endpoint_deployments" } 2015-10-29T01:36:55.322Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.331Z I INDEX [rsSync] build index done. scanned 236 total records. 0 secs 2015-10-29T01:36:55.334Z I INDEX [rsSync] build index on: endpoint_mcs.endpoint_installers properties: { v: 1, key: { deployment_id: 1 }, name: "deployment_id_1", background: true, ns: "endpoint_mcs.endpoint_installers" } 2015-10-29T01:36:55.334Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.336Z I INDEX [rsSync] build index on: endpoint_mcs.endpoint_installers properties: { v: 1, unique: true, key: { lt: 1 }, name: "lt_1", sparse: false, ns: "endpoint_mcs.endpoint_installers" } 2015-10-29T01:36:55.336Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.345Z I INDEX [rsSync] build index done. scanned 472 total records. 0 secs 2015-10-29T01:36:55.349Z I INDEX [rsSync] build index on: endpoint_mcs.endpoints properties: { v: 1, key: { customer_id: 1, domain_name: 1, computer_name: 1, operating_system: 1 }, name: "customer_id_1_domain_name_1_computer_name_1_operating_system_1", ns: "endpoint_mcs.endpoints" } 2015-10-29T01:36:55.349Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.351Z I INDEX [rsSync] build index on: endpoint_mcs.endpoints properties: { v: 1, key: { active: 1, last_active: 1 }, name: "active_1_last_active_1", ns: "endpoint_mcs.endpoints" } 2015-10-29T01:36:55.351Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.375Z I INDEX [rsSync] build index done. scanned 2385 total records. 0 secs 2015-10-29T01:36:55.378Z I INDEX [rsSync] build index on: endpoint_mcs.ham_mq_envelopes properties: { v: 1, key: { queue_id: 1, state: 1, sequence_id: 1 }, name: "queue_id_1_state_1_sequence_id_1", ns: "endpoint_mcs.ham_mq_envelopes" } 2015-10-29T01:36:55.379Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.381Z I INDEX [rsSync] build index on: endpoint_mcs.ham_mq_envelopes properties: { v: 1, key: { queue_id: 1, expires_at: 1 }, name: "queue_id_1_expires_at_1", ns: "endpoint_mcs.ham_mq_envelopes" } 2015-10-29T01:36:55.381Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.383Z I INDEX [rsSync] build index on: endpoint_mcs.ham_mq_envelopes properties: { v: 1, unique: true, key: { queue_id: 1, unique_id: 1 }, name: "queue_id_1_unique_id_1", ns: "endpoint_mcs.ham_mq_envelopes" } 2015-10-29T01:36:55.383Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.392Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:55.395Z I INDEX [rsSync] build index on: endpoint_mcs.ham_mq_group_locks properties: { v: 1, unique: true, key: { queue_id: 1, group_id: 1 }, name: "queue_id_1_group_id_1", ns: "endpoint_mcs.ham_mq_group_locks" } 2015-10-29T01:36:55.395Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.397Z I INDEX [rsSync] build index on: endpoint_mcs.ham_mq_group_locks properties: { v: 1, key: { queue_id: 1, created_at: 1 }, name: "queue_id_1_created_at_1", ns: "endpoint_mcs.ham_mq_group_locks" } 2015-10-29T01:36:55.397Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.399Z I INDEX [rsSync] build index on: endpoint_mcs.ham_mq_group_locks properties: { v: 1, key: { queue_id: 1, worker_id: 1, group_id: 1 }, name: "queue_id_1_worker_id_1_group_id_1", ns: "endpoint_mcs.ham_mq_group_locks" } 2015-10-29T01:36:55.399Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.407Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:55.408Z I REPL [rsSync] initial sync cloning indexes for : htest 2015-10-29T01:36:55.438Z I REPL [rsSync] initial sync cloning indexes for : mobile 2015-10-29T01:36:55.474Z I INDEX [rsSync] build index on: mobile.app_data properties: { v: 1, key: { installation_id: 1 }, name: "installation_id_1", ns: "mobile.app_data", sparse: false, background: true } 2015-10-29T01:36:55.474Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.476Z I INDEX [rsSync] build index on: mobile.app_data properties: { v: 1, key: { identifier: 1 }, name: "identifier_1", ns: "mobile.app_data", sparse: false, background: true } 2015-10-29T01:36:55.476Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.482Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:55.486Z I INDEX [rsSync] build index on: mobile.device_actions properties: { v: 1, key: { device_id: 1 }, name: "device_id_1", ns: "mobile.device_actions", background: true } 2015-10-29T01:36:55.486Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.489Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:55.492Z I INDEX [rsSync] build index on: mobile.device_models properties: { v: 1, unique: true, key: { key: 1, platform: 1 }, name: "key_1_platform_1", ns: "mobile.device_models", sparse: false, background: true } 2015-10-29T01:36:55.492Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.518Z I INDEX [rsSync] build index done. scanned 7912 total records. 0 secs 2015-10-29T01:36:55.521Z I INDEX [rsSync] build index on: mobile.devices properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "mobile.devices", sparse: false, background: true } 2015-10-29T01:36:55.521Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.523Z I INDEX [rsSync] build index on: mobile.devices properties: { v: 1, unique: true, key: { device_id: 1 }, name: "device_id_1", ns: "mobile.devices", sparse: true, background: true } 2015-10-29T01:36:55.523Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.526Z I INDEX [rsSync] build index on: mobile.devices properties: { v: 1, key: { endpoint_id: 1 }, name: "endpoint_id_1", ns: "mobile.devices", sparse: false, background: true } 2015-10-29T01:36:55.526Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.528Z I INDEX [rsSync] build index on: mobile.devices properties: { v: 1, key: { enrollment_id: 1 }, name: "enrollment_id_1", ns: "mobile.devices", sparse: false, background: true } 2015-10-29T01:36:55.528Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.531Z I INDEX [rsSync] build index on: mobile.devices properties: { v: 1, key: { operating_system_id: 1 }, name: "operating_system_id_1", ns: "mobile.devices", sparse: false, background: true } 2015-10-29T01:36:55.531Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.533Z I INDEX [rsSync] build index on: mobile.devices properties: { v: 1, unique: true, key: { hardware_id: 1 }, name: "hardware_id_1", ns: "mobile.devices", sparse: true, background: true } 2015-10-29T01:36:55.533Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.552Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:55.555Z I INDEX [rsSync] build index on: mobile.enrollments properties: { v: 1, unique: true, key: { token: 1 }, name: "token_1", ns: "mobile.enrollments", sparse: true, background: true } 2015-10-29T01:36:55.555Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.557Z I INDEX [rsSync] build index on: mobile.enrollments properties: { v: 1, key: { registration_token: 1 }, name: "registration_token_1", ns: "mobile.enrollments", sparse: false, background: true } 2015-10-29T01:36:55.558Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.560Z I INDEX [rsSync] build index on: mobile.enrollments properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "mobile.enrollments", sparse: false, background: true } 2015-10-29T01:36:55.560Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.569Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:55.572Z I INDEX [rsSync] build index on: mobile.ham_mq_envelopes properties: { v: 1, key: { queue_id: 1, state: 1, sequence_id: 1 }, name: "queue_id_1_state_1_sequence_id_1", ns: "mobile.ham_mq_envelopes" } 2015-10-29T01:36:55.572Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.574Z I INDEX [rsSync] build index on: mobile.ham_mq_envelopes properties: { v: 1, key: { queue_id: 1, expires_at: 1 }, name: "queue_id_1_expires_at_1", ns: "mobile.ham_mq_envelopes" } 2015-10-29T01:36:55.574Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.576Z I INDEX [rsSync] build index on: mobile.ham_mq_envelopes properties: { v: 1, unique: true, key: { queue_id: 1, unique_id: 1 }, name: "queue_id_1_unique_id_1", ns: "mobile.ham_mq_envelopes" } 2015-10-29T01:36:55.576Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.584Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:55.587Z I INDEX [rsSync] build index on: mobile.ham_mq_group_locks properties: { v: 1, unique: true, key: { queue_id: 1, group_id: 1 }, name: "queue_id_1_group_id_1", ns: "mobile.ham_mq_group_locks" } 2015-10-29T01:36:55.587Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.590Z I INDEX [rsSync] build index on: mobile.ham_mq_group_locks properties: { v: 1, key: { queue_id: 1, created_at: 1 }, name: "queue_id_1_created_at_1", ns: "mobile.ham_mq_group_locks" } 2015-10-29T01:36:55.590Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.592Z I INDEX [rsSync] build index on: mobile.ham_mq_group_locks properties: { v: 1, key: { queue_id: 1, worker_id: 1, group_id: 1 }, name: "queue_id_1_worker_id_1_group_id_1", ns: "mobile.ham_mq_group_locks" } 2015-10-29T01:36:55.592Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.600Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:55.601Z I REPL [rsSync] initial sync cloning indexes for : mobile_dev 2015-10-29T01:36:55.633Z I INDEX [rsSync] build index on: mobile_dev.device_models properties: { v: 1, unique: true, key: { key: 1, platform: 1 }, name: "key_1_platform_1", ns: "mobile_dev.device_models", sparse: false, background: false } 2015-10-29T01:36:55.633Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.636Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:55.639Z I INDEX [rsSync] build index on: mobile_dev.devices properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "mobile_dev.devices", sparse: false, background: false } 2015-10-29T01:36:55.639Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.641Z I INDEX [rsSync] build index on: mobile_dev.devices properties: { v: 1, unique: true, key: { device_id: 1 }, name: "device_id_1", ns: "mobile_dev.devices", sparse: true, background: false } 2015-10-29T01:36:55.641Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.643Z I INDEX [rsSync] build index on: mobile_dev.devices properties: { v: 1, key: { endpoint_id: 1 }, name: "endpoint_id_1", ns: "mobile_dev.devices", sparse: false, background: false } 2015-10-29T01:36:55.643Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.647Z I INDEX [rsSync] build index on: mobile_dev.devices properties: { v: 1, key: { enrollment_id: 1 }, name: "enrollment_id_1", ns: "mobile_dev.devices", sparse: false, background: false } 2015-10-29T01:36:55.647Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.649Z I INDEX [rsSync] build index on: mobile_dev.devices properties: { v: 1, key: { operating_system_id: 1 }, name: "operating_system_id_1", ns: "mobile_dev.devices", sparse: false, background: false } 2015-10-29T01:36:55.649Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.664Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:55.667Z I INDEX [rsSync] build index on: mobile_dev.enrollments properties: { v: 1, unique: true, key: { token: 1 }, name: "token_1", ns: "mobile_dev.enrollments", sparse: true, background: false } 2015-10-29T01:36:55.667Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.669Z I INDEX [rsSync] build index on: mobile_dev.enrollments properties: { v: 1, key: { registration_token: 1 }, name: "registration_token_1", ns: "mobile_dev.enrollments", sparse: false, background: false } 2015-10-29T01:36:55.669Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.671Z I INDEX [rsSync] build index on: mobile_dev.enrollments properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "mobile_dev.enrollments", sparse: false, background: false } 2015-10-29T01:36:55.671Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.680Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:36:55.681Z I REPL [rsSync] initial sync cloning indexes for : scalability 2015-10-29T01:36:55.716Z I REPL [rsSync] initial sync cloning indexes for : sledge 2015-10-29T01:36:55.749Z I INDEX [rsSync] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, hidden: 1, alert_type: 1 }, name: "customer_id_1_hidden_1_alert_type_1", ns: "sledge.alerts", background: true } 2015-10-29T01:36:55.749Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.751Z I INDEX [rsSync] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, unhide_at: 1 }, name: "customer_id_1_unhide_at_1", ns: "sledge.alerts", background: true } 2015-10-29T01:36:55.751Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.754Z I INDEX [rsSync] build index on: sledge.alerts properties: { v: 1, key: { unhide_at: 1, hidden: 1 }, name: "unhide_at_1_hidden_1", ns: "sledge.alerts", background: true } 2015-10-29T01:36:55.754Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.756Z I INDEX [rsSync] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, data.endpoint_id: 1, alert_type: 1 }, name: "customer_id_1_data.endpoint_id_1_alert_type_1", ns: "sledge.alerts", background: true } 2015-10-29T01:36:55.756Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.759Z I INDEX [rsSync] build index on: sledge.alerts properties: { v: 1, key: { customer_id: 1, data.threat_id: 1, alert_type: 1 }, name: "customer_id_1_data.threat_id_1_alert_type_1", ns: "sledge.alerts", background: true } 2015-10-29T01:36:55.759Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.761Z I INDEX [rsSync] build index on: sledge.alerts properties: { v: 1, key: { endpoint_id: 1 }, name: "endpoint_id_1", ns: "sledge.alerts", background: true, sparse: true } 2015-10-29T01:36:55.761Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.782Z I INDEX [rsSync] build index done. scanned 9 total records. 0 secs 2015-10-29T01:36:55.786Z I INDEX [rsSync] build index on: sledge.business_intelligence_reports properties: { v: 1, key: { report_time: 1 }, name: "report_time_1", ns: "sledge.business_intelligence_reports", background: true } 2015-10-29T01:36:55.786Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.790Z I INDEX [rsSync] build index done. scanned 72 total records. 0 secs 2015-10-29T01:36:55.793Z I INDEX [rsSync] build index on: sledge.controlled_applications properties: { v: 1, key: { category_id: 1 }, name: "category_id_1", ns: "sledge.controlled_applications", background: true } 2015-10-29T01:36:55.793Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.800Z I INDEX [rsSync] build index done. scanned 1514 total records. 0 secs 2015-10-29T01:36:55.803Z I INDEX [rsSync] build index on: sledge.customer_flags properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.customer_flags", sparse: false, background: true } 2015-10-29T01:36:55.803Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.907Z I INDEX [rsSync] build index done. scanned 43290 total records. 0 secs 2015-10-29T01:36:55.912Z I INDEX [rsSync] build index on: sledge.customers properties: { v: 1, key: { licenses.expires_on: 1 }, name: "licenses.expires_on_1", ns: "sledge.customers", sparse: false, background: true } 2015-10-29T01:36:55.912Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.914Z I INDEX [rsSync] build index on: sledge.customers properties: { v: 1, key: { beta: 1 }, name: "beta_1", ns: "sledge.customers", sparse: true, background: true } 2015-10-29T01:36:55.914Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.933Z I INDEX [rsSync] build index done. scanned 2405 total records. 0 secs 2015-10-29T01:36:55.937Z I INDEX [rsSync] build index on: sledge.deployment_tokens properties: { v: 1, unique: true, key: { registration_token: 1 }, name: "registration_token_1", ns: "sledge.deployment_tokens", background: true } 2015-10-29T01:36:55.937Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.939Z I INDEX [rsSync] build index on: sledge.deployment_tokens properties: { v: 1, key: { user_id: 1 }, name: "user_id_1", ns: "sledge.deployment_tokens", background: true } 2015-10-29T01:36:55.939Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.941Z I INDEX [rsSync] build index on: sledge.deployment_tokens properties: { v: 1, key: { customer_id: 1, user_id: 1 }, name: "customer_id_1_user_id_1", ns: "sledge.deployment_tokens", background: true } 2015-10-29T01:36:55.941Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.944Z I INDEX [rsSync] build index on: sledge.deployment_tokens properties: { v: 1, key: { customer_id: 1, user_id: 1, beta: 1 }, name: "customer_id_1_user_id_1_beta_1", ns: "sledge.deployment_tokens", sparse: false, background: true } 2015-10-29T01:36:55.944Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:55.999Z I INDEX [rsSync] build index done. scanned 4387 total records. 0 secs 2015-10-29T01:36:56.002Z I INDEX [rsSync] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1 }, name: "c_1__type_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-29T01:36:56.002Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:56.005Z I INDEX [rsSync] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, name_lowercase: 1 }, name: "c_1__type_1_name_lowercase_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-29T01:36:56.005Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:56.007Z I INDEX [rsSync] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, search_tokens: 1 }, name: "c_1__type_1_search_tokens_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-29T01:36:56.007Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:56.009Z I INDEX [rsSync] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, external_id: 1 }, name: "c_1__type_1_external_id_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-29T01:36:56.009Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:56.012Z I INDEX [rsSync] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, origin: 1 }, name: "c_1__type_1_origin_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-29T01:36:56.012Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:56.014Z I INDEX [rsSync] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, email: 1 }, name: "c_1__type_1_email_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-29T01:36:56.014Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:56.016Z I INDEX [rsSync] build index on: sledge.directory_objects properties: { v: 1, key: { c: 1, _type: 1, dn: 1 }, name: "c_1__type_1_dn_1", ns: "sledge.directory_objects", sparse: false, background: true } 2015-10-29T01:36:56.016Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:36:59.003Z I - [rsSync] Index Build: 311300/327067 95% 2015-10-29T01:37:07.052Z I INDEX [rsSync] build index done. scanned 327067 total records. 11 secs 2015-10-29T01:37:07.277Z I INDEX [rsSync] build index on: sledge.directory_user_matches properties: { v: 1, key: { local_endpoint_id: 1 }, name: "local_endpoint_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-29T01:37:07.277Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:07.279Z I INDEX [rsSync] build index on: sledge.directory_user_matches properties: { v: 1, key: { name_lowercase: 1 }, name: "name_lowercase_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-29T01:37:07.279Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:07.281Z I INDEX [rsSync] build index on: sledge.directory_user_matches properties: { v: 1, unique: true, key: { name_lowercase: 1, c: 1 }, name: "name_lowercase_1_c_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-29T01:37:07.281Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:07.283Z I INDEX [rsSync] build index on: sledge.directory_user_matches properties: { v: 1, key: { user_id: 1 }, name: "user_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-29T01:37:07.283Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:07.286Z I INDEX [rsSync] build index on: sledge.directory_user_matches properties: { v: 1, key: { customer_id: 1, name_lowercase: 1 }, name: "customer_id_1_name_lowercase_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-29T01:37:07.286Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:07.289Z I INDEX [rsSync] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, local_endpoint_id: 1 }, name: "c_1_local_endpoint_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-29T01:37:07.290Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:07.292Z I INDEX [rsSync] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, name_lowercase: 1 }, name: "c_1_name_lowercase_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-29T01:37:07.292Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:07.294Z I INDEX [rsSync] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, external_id: 1 }, name: "c_1_external_id_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-29T01:37:07.294Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:07.296Z I INDEX [rsSync] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, origin: 1 }, name: "c_1_origin_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-29T01:37:07.296Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:07.299Z I INDEX [rsSync] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, name: 1 }, name: "c_1_name_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-29T01:37:07.299Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:07.301Z I INDEX [rsSync] build index on: sledge.directory_user_matches properties: { v: 1, key: { c: 1, dn: 1 }, name: "c_1_dn_1", ns: "sledge.directory_user_matches", sparse: false, background: true } 2015-10-29T01:37:07.301Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:10.002Z I - [rsSync] Index Build: 317900/336027 94% 2015-10-29T01:37:19.438Z I INDEX [rsSync] build index done. scanned 336027 total records. 12 secs 2015-10-29T01:37:19.868Z I INDEX [rsSync] build index on: sledge.endpoint_credentials properties: { v: 1, key: { endpoint_id: 1 }, name: "endpoint_id_1", ns: "sledge.endpoint_credentials", background: true } 2015-10-29T01:37:19.868Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:19.870Z I INDEX [rsSync] build index on: sledge.endpoint_credentials properties: { v: 1, key: { customer_id: 1, transport: 1 }, name: "customer_id_1_transport_1", ns: "sledge.endpoint_credentials", background: true } 2015-10-29T01:37:19.870Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:19.876Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:37:19.879Z I INDEX [rsSync] build index on: sledge.endpoint_devices properties: { v: 1, key: { customer_id: 1, detected_at: 1 }, name: "customer_id_1_detected_at_1", ns: "sledge.endpoint_devices", background: true } 2015-10-29T01:37:19.879Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:19.881Z I INDEX [rsSync] build index on: sledge.endpoint_devices properties: { v: 1, key: { endpoint_id: 1, device_id: 1, instance_id: 1 }, name: "endpoint_id_1_device_id_1_instance_id_1", ns: "sledge.endpoint_devices", background: true } 2015-10-29T01:37:19.881Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:19.884Z I INDEX [rsSync] build index on: sledge.endpoint_devices properties: { v: 1, key: { endpoint_id: 1, device_id: 1, device_type: 1, instance_id: 1, description: 1, detected_at: 1 }, name: "endpoint_id_1_device_id_1_device_type_1_instance_id_1_description_1_detected_at_1", ns: "sledge.endpoint_devices", background: true } 2015-10-29T01:37:19.884Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:19.886Z I INDEX [rsSync] build index on: sledge.endpoint_devices properties: { v: 1, key: { customer_id: 1, device_type: 1, detected_at: 1 }, name: "customer_id_1_device_type_1_detected_at_1", ns: "sledge.endpoint_devices", background: true } 2015-10-29T01:37:19.886Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:22.422Z I INDEX [rsSync] build index done. scanned 147812 total records. 2 secs 2015-10-29T01:37:22.497Z I INDEX [rsSync] build index on: sledge.endpoint_threats properties: { v: 1, key: { endpoint_id: 1, threat_id: 1 }, name: "endpoint_id_1_threat_id_1", ns: "sledge.endpoint_threats", background: true } 2015-10-29T01:37:22.497Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:22.499Z I INDEX [rsSync] build index on: sledge.endpoint_threats properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.endpoint_threats", background: true } 2015-10-29T01:37:22.499Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:22.973Z I INDEX [rsSync] build index done. scanned 76203 total records. 0 secs 2015-10-29T01:37:22.992Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { user_match_id: 1 }, name: "user_match_id_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-29T01:37:22.992Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:22.994Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { domain_name: 1, computer_name: 1, operating_system: 1 }, name: "domain_name_1_computer_name_1_operating_system_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-29T01:37:22.994Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:22.996Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-29T01:37:22.996Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:22.998Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, feature_codes: 1 }, name: "customer_id_1_feature_codes_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-29T01:37:22.998Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:23.000Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, label: 1 }, name: "customer_id_1_label_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-29T01:37:23.000Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:23.003Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, label_lowercase: 1 }, name: "customer_id_1_label_lowercase_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-29T01:37:23.003Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:23.005Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, user_match_id: 1 }, name: "customer_id_1_user_match_id_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-29T01:37:23.005Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:23.007Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, search_tokens: 1 }, name: "customer_id_1_search_tokens_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-29T01:37:23.007Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:23.009Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, transport: 1 }, name: "customer_id_1_transport_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-29T01:37:23.009Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:23.011Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, endpoint_type: 1 }, name: "customer_id_1_endpoint_type_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-29T01:37:23.011Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:23.014Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, info.platform: 1 }, name: "customer_id_1_info.platform_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-29T01:37:23.014Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:23.016Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, registered_at: 1 }, name: "customer_id_1_registered_at_1", ns: "sledge.endpoints", sparse: false, background: true } 2015-10-29T01:37:23.016Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:23.018Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { last_connected_from: 1 }, name: "last_connected_from_1", ns: "sledge.endpoints", sparse: true, background: true } 2015-10-29T01:37:23.018Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:23.021Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { beta: 1 }, name: "beta_1", ns: "sledge.endpoints", sparse: true, background: true } 2015-10-29T01:37:23.021Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:23.023Z I INDEX [rsSync] build index on: sledge.endpoints properties: { v: 1, key: { customer_id: 1, info.isSecurityVm: 1 }, name: "customer_id_1_info.isSecurityVm_1", ns: "sledge.endpoints", background: true } 2015-10-29T01:37:23.023Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:26.003Z I - [rsSync] Index Build: 139400/221260 63% 2015-10-29T01:37:35.159Z I INDEX [rsSync] build index done. scanned 221260 total records. 12 secs 2015-10-29T01:37:35.469Z I INDEX [rsSync] build index on: sledge.event_bases properties: { v: 1, key: { customer_id: 1, severity: 1, reported_at: 1, _type: 1 }, name: "customer_id_1_severity_1_reported_at_1__type_1", ns: "sledge.event_bases", sparse: false, background: true } 2015-10-29T01:37:35.469Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:35.471Z I INDEX [rsSync] build index on: sledge.event_bases properties: { v: 1, key: { customer_id: 1, reported_at: -1, _type: 1 }, name: "customer_id_1_reported_at_-1__type_1", ns: "sledge.event_bases", sparse: false, background: true } 2015-10-29T01:37:35.471Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:35.474Z I INDEX [rsSync] build index on: sledge.event_bases properties: { v: 1, key: { _type: 1, reported_at: 1 }, name: "_type_1_reported_at_1", ns: "sledge.event_bases", sparse: false, background: true } 2015-10-29T01:37:35.474Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:35.476Z I INDEX [rsSync] build index on: sledge.event_bases properties: { v: 1, key: { reported_at: 1 }, name: "reported_at_1", ns: "sledge.event_bases", sparse: false, background: true } 2015-10-29T01:37:35.476Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:35.478Z I INDEX [rsSync] build index on: sledge.event_bases properties: { v: 1, key: { customer_id: 1, inserted_at: 1 }, name: "customer_id_1_inserted_at_1", ns: "sledge.event_bases", sparse: true, background: true } 2015-10-29T01:37:35.478Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:35.481Z I INDEX [rsSync] build index on: sledge.event_bases properties: { v: 1, key: { endpoint_id: 1, _type: 1, reported_at: 1 }, name: "endpoint_id_1__type_1_reported_at_1", ns: "sledge.event_bases", sparse: false, background: true } 2015-10-29T01:37:35.481Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:35.483Z I INDEX [rsSync] build index on: sledge.event_bases properties: { v: 1, key: { user_match_id: 1 }, name: "user_match_id_1", ns: "sledge.event_bases", sparse: true, background: true } 2015-10-29T01:37:35.483Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:35.485Z I INDEX [rsSync] build index on: sledge.event_bases properties: { v: 1, key: { threat_id: 1, _type: 1 }, name: "threat_id_1__type_1", ns: "sledge.event_bases", background: true } 2015-10-29T01:37:35.485Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:35.488Z I INDEX [rsSync] build index on: sledge.event_bases properties: { v: 1, key: { device_id: 1, _type: 1, reported_at: 1 }, name: "device_id_1__type_1_reported_at_1", ns: "sledge.event_bases", background: true } 2015-10-29T01:37:35.488Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:35.490Z I INDEX [rsSync] build index on: sledge.event_bases properties: { v: 1, key: { device_id: 1, reported_at: -1 }, name: "device_id_1_reported_at_-1", ns: "sledge.event_bases", background: true } 2015-10-29T01:37:35.490Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:35.492Z I INDEX [rsSync] build index on: sledge.event_bases properties: { v: 1, key: { customer_id: 1, device_id: 1 }, name: "customer_id_1_device_id_1", ns: "sledge.event_bases", background: true } 2015-10-29T01:37:35.492Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:37:38.002Z I - [rsSync] Index Build: 209800/5520917 3% 2015-10-29T01:37:41.002Z I - [rsSync] Index Build: 463300/5520917 8% 2015-10-29T01:37:44.002Z I - [rsSync] Index Build: 712500/5520917 12% 2015-10-29T01:37:47.003Z I - [rsSync] Index Build: 961700/5520917 17% 2015-10-29T01:37:50.232Z I - [rsSync] Index Build: 1023200/5520917 18% 2015-10-29T01:37:56.929Z I - [rsSync] Index Build: 1084000/5520917 19% 2015-10-29T01:37:59.002Z I - [rsSync] Index Build: 1251300/5520917 22% 2015-10-29T01:38:04.094Z I - [rsSync] Index Build: 1363900/5520917 24% 2015-10-29T01:38:07.422Z I - [rsSync] Index Build: 1403800/5520917 25% 2015-10-29T01:38:12.939Z I - [rsSync] Index Build: 1556900/5520917 28% 2015-10-29T01:38:18.763Z I - [rsSync] Index Build: 1691300/5520917 30% 2015-10-29T01:38:21.002Z I - [rsSync] Index Build: 1871700/5520917 33% 2015-10-29T01:38:26.819Z I - [rsSync] Index Build: 1928400/5520917 34% 2015-10-29T01:38:31.005Z I - [rsSync] Index Build: 2003500/5520917 36% 2015-10-29T01:38:39.164Z I - [rsSync] Index Build: 2122500/5520917 38% 2015-10-29T01:38:42.002Z I - [rsSync] Index Build: 2351100/5520917 42% 2015-10-29T01:38:47.882Z I - [rsSync] Index Build: 2438600/5520917 44% 2015-10-29T01:38:55.059Z I - [rsSync] Index Build: 2531300/5520917 45% 2015-10-29T01:39:01.126Z I - [rsSync] Index Build: 2678400/5520917 48% 2015-10-29T01:39:04.471Z I - [rsSync] Index Build: 2688700/5520917 48% 2015-10-29T01:39:08.708Z I - [rsSync] Index Build: 2761200/5520917 50% 2015-10-29T01:39:11.003Z I - [rsSync] Index Build: 2945700/5520917 53% 2015-10-29T01:39:14.747Z I - [rsSync] Index Build: 2992900/5520917 54% 2015-10-29T01:39:19.943Z I - [rsSync] Index Build: 3057400/5520917 55% 2015-10-29T01:39:27.931Z I - [rsSync] Index Build: 3170600/5520917 57% 2015-10-29T01:39:30.002Z I - [rsSync] Index Build: 3336300/5520917 60% 2015-10-29T01:39:35.342Z I - [rsSync] Index Build: 3382600/5520917 61% 2015-10-29T01:39:38.003Z I - [rsSync] Index Build: 3592700/5520917 65% 2015-10-29T01:39:41.002Z I - [rsSync] Index Build: 3833500/5520917 69% 2015-10-29T01:39:47.006Z I - [rsSync] Index Build: 3849700/5520917 69% 2015-10-29T01:39:51.895Z I - [rsSync] Index Build: 3973000/5520917 71% 2015-10-29T01:39:56.366Z I - [rsSync] Index Build: 3979200/5520917 72% 2015-10-29T01:40:01.623Z I - [rsSync] Index Build: 4105800/5520917 74% 2015-10-29T01:40:09.651Z I - [rsSync] Index Build: 4209100/5520917 76% 2015-10-29T01:40:12.002Z I - [rsSync] Index Build: 4396700/5520917 79% 2015-10-29T01:40:18.485Z I - [rsSync] Index Build: 4542300/5520917 82% 2015-10-29T01:40:21.002Z I - [rsSync] Index Build: 4742700/5520917 85% 2015-10-29T01:40:28.207Z I - [rsSync] Index Build: 4877200/5520917 88% 2015-10-29T01:40:32.541Z I - [rsSync] Index Build: 4953200/5520917 89% 2015-10-29T01:40:40.800Z I - [rsSync] Index Build: 5052000/5520917 91% 2015-10-29T01:40:45.946Z I - [rsSync] Index Build: 5073800/5520917 91% 2015-10-29T01:40:48.002Z I - [rsSync] Index Build: 5235800/5520917 94% 2015-10-29T01:40:54.957Z I - [rsSync] Index Build: 5250600/5520917 95% 2015-10-29T01:40:59.897Z I - [rsSync] Index Build: 5284300/5520917 95% 2015-10-29T01:41:04.670Z I - [rsSync] Index Build: 5377400/5520917 97% 2015-10-29T01:41:09.348Z I - [rsSync] Index Build: 5454400/5520917 98% 2015-10-29T01:42:11.460Z I INDEX [rsSync] build index done. scanned 5520917 total records. 275 secs 2015-10-29T01:42:11.540Z I INDEX [rsSync] build index on: sledge.global_exclusions properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.global_exclusions", sparse: false, background: true } 2015-10-29T01:42:11.540Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:11.550Z I INDEX [rsSync] build index done. scanned 2405 total records. 0 secs 2015-10-29T01:42:11.554Z I INDEX [rsSync] build index on: sledge.home_intelligence_reports properties: { v: 1, key: { report_time: 1 }, name: "report_time_1", ns: "sledge.home_intelligence_reports", background: true } 2015-10-29T01:42:11.554Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:11.558Z I INDEX [rsSync] build index done. scanned 62 total records. 0 secs 2015-10-29T01:42:11.561Z I INDEX [rsSync] build index on: sledge.local_sites properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.local_sites", background: true } 2015-10-29T01:42:11.561Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:11.563Z I INDEX [rsSync] build index on: sledge.local_sites properties: { v: 1, key: { customer_id: 1, url: 1 }, name: "customer_id_1_url_1", ns: "sledge.local_sites", background: true } 2015-10-29T01:42:11.563Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:11.568Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:42:11.571Z I INDEX [rsSync] build index on: sledge.mcs_commands properties: { v: 1, key: { endpoint_id: 1, command_id: 1 }, name: "endpoint_id_1_command_id_1", ns: "sledge.mcs_commands", background: true } 2015-10-29T01:42:11.571Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:11.573Z I INDEX [rsSync] build index on: sledge.mcs_commands properties: { v: 1, key: { endpoint_id: 1, command_app_id: 1 }, name: "endpoint_id_1_command_app_id_1", ns: "sledge.mcs_commands", background: true } 2015-10-29T01:42:11.573Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:11.576Z I INDEX [rsSync] build index on: sledge.mcs_commands properties: { v: 1, key: { endpoint_id: 1, posted_app_id: 1, body_sha: 1 }, name: "endpoint_id_1_posted_app_id_1_body_sha_1", ns: "sledge.mcs_commands", background: true } 2015-10-29T01:42:11.576Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:11.578Z I INDEX [rsSync] build index on: sledge.mcs_commands properties: { v: 1, key: { created_at: 1 }, name: "created_at_1", ns: "sledge.mcs_commands", background: true } 2015-10-29T01:42:11.578Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:13.923Z I INDEX [rsSync] build index done. scanned 192792 total records. 2 secs 2015-10-29T01:42:14.037Z I INDEX [rsSync] build index on: sledge.mcs_policy_bases properties: { v: 1, key: { customer_id: 1, _type: 1 }, name: "customer_id_1__type_1", ns: "sledge.mcs_policy_bases", sparse: false, background: true } 2015-10-29T01:42:14.037Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:14.073Z I INDEX [rsSync] build index done. scanned 9620 total records. 0 secs 2015-10-29T01:42:14.077Z I INDEX [rsSync] build index on: sledge.message_schedules properties: { v: 1, key: { scheduled_at: 1 }, name: "scheduled_at_1", ns: "sledge.message_schedules", background: true } 2015-10-29T01:42:14.077Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:14.082Z I INDEX [rsSync] build index done. scanned 14 total records. 0 secs 2015-10-29T01:42:14.085Z I INDEX [rsSync] build index on: sledge.messages properties: { v: 1, key: { created_at: 1, picked_at: 1 }, name: "created_at_1_picked_at_1", ns: "sledge.messages", background: true } 2015-10-29T01:42:14.085Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:14.087Z I INDEX [rsSync] build index on: sledge.messages properties: { v: 1, key: { queue: 1 }, name: "queue_1", ns: "sledge.messages", background: true } 2015-10-29T01:42:14.087Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:14.089Z I INDEX [rsSync] build index on: sledge.messages properties: { v: 1, unique: true, key: { key: 1 }, name: "key_1", ns: "sledge.messages", background: true, sparse: true } 2015-10-29T01:42:14.089Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:14.098Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:42:14.101Z I INDEX [rsSync] build index on: sledge.policies properties: { v: 1, key: { customer_id: 1, target: 1, priority: -1 }, name: "customer_id_1_target_1_priority_-1", ns: "sledge.policies", sparse: false, background: true } 2015-10-29T01:42:14.101Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:14.103Z I INDEX [rsSync] build index on: sledge.policies properties: { v: 1, key: { enabled: 1, disable_at: 1 }, name: "enabled_1_disable_at_1", ns: "sledge.policies", sparse: false, background: true } 2015-10-29T01:42:14.103Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:14.209Z I INDEX [rsSync] build index done. scanned 15490 total records. 0 secs 2015-10-29T01:42:14.215Z I INDEX [rsSync] build index on: sledge.policy_assignments properties: { v: 1, key: { endpoint_id: 1, policy_type: 1 }, name: "endpoint_id_1_policy_type_1", ns: "sledge.policy_assignments", sparse: false, background: true } 2015-10-29T01:42:14.215Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:14.217Z I INDEX [rsSync] build index on: sledge.policy_assignments properties: { v: 1, key: { endpoint_id: 1 }, name: "endpoint_id_1", ns: "sledge.policy_assignments", sparse: false, background: true } 2015-10-29T01:42:14.217Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:17.002Z I - [rsSync] Index Build: 1430300/2202980 64% 2015-10-29T01:42:21.312Z I - [rsSync] Index Build: 1807900/2202980 82% 2015-10-29T01:42:25.441Z I - [rsSync] Index Build: 2016500/2202980 91% 2015-10-29T01:42:29.461Z I INDEX [rsSync] build index done. scanned 2202980 total records. 15 secs 2015-10-29T01:42:29.470Z I INDEX [rsSync] build index on: sledge.rendered_policies properties: { v: 1, key: { policy_fragment_ids: 1 }, name: "policy_fragment_ids_1", ns: "sledge.rendered_policies", sparse: false, background: true } 2015-10-29T01:42:29.470Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.599Z I INDEX [rsSync] build index done. scanned 27395 total records. 0 secs 2015-10-29T01:42:29.606Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "sledge.report_bases", background: true } 2015-10-29T01:42:29.606Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.608Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1 }, name: "customer_id_1__type_1", ns: "sledge.report_bases", background: true } 2015-10-29T01:42:29.608Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.610Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { ham_idx_upd: -1 }, name: "ham_idx_upd_-1", ns: "sledge.report_bases", background: true, sparse: true } 2015-10-29T01:42:29.610Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.612Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, tm: 1 }, name: "customer_id_1__type_1_tm_1", ns: "sledge.report_bases", background: true } 2015-10-29T01:42:29.613Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.615Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, dom: 1 }, name: "customer_id_1_dom_1", ns: "sledge.report_bases", background: true } 2015-10-29T01:42:29.615Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.618Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, rsn: 1 }, name: "customer_id_1_rsn_1", ns: "sledge.report_bases", background: true } 2015-10-29T01:42:29.618Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.620Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, cat: 1 }, name: "customer_id_1_cat_1", ns: "sledge.report_bases", background: true } 2015-10-29T01:42:29.620Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.622Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, hits.type: 1, dom: 1 }, name: "customer_id_1_hits.type_1_dom_1", ns: "sledge.report_bases", background: true } 2015-10-29T01:42:29.622Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.625Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, dom: 1, cat: 1, rsn: 1 }, name: "customer_id_1__type_1_dom_1_cat_1_rsn_1", ns: "sledge.report_bases", background: true } 2015-10-29T01:42:29.625Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.627Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, name: 1, group: 1 }, name: "customer_id_1_name_1_group_1", ns: "sledge.report_bases", background: true } 2015-10-29T01:42:29.627Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.629Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, hits.type: 1, name: 1, group: 1 }, name: "customer_id_1_hits.type_1_name_1_group_1", ns: "sledge.report_bases", background: true } 2015-10-29T01:42:29.629Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.631Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, name: 1 }, name: "customer_id_1__type_1_name_1", ns: "sledge.report_bases", background: true } 2015-10-29T01:42:29.631Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.634Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, usr: 1 }, name: "customer_id_1_usr_1", ns: "sledge.report_bases", background: true } 2015-10-29T01:42:29.634Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.636Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, loc: 1 }, name: "customer_id_1_loc_1", ns: "sledge.report_bases", background: true } 2015-10-29T01:42:29.636Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.640Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, hits.type: 1, usr: 1 }, name: "customer_id_1_hits.type_1_usr_1", ns: "sledge.report_bases", background: true } 2015-10-29T01:42:29.640Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:29.642Z I INDEX [rsSync] build index on: sledge.report_bases properties: { v: 1, key: { customer_id: 1, _type: 1, usr: 1, loc: 1 }, name: "customer_id_1__type_1_usr_1_loc_1", ns: "sledge.report_bases", background: true } 2015-10-29T01:42:29.642Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:32.002Z I - [rsSync] Index Build: 92000/292160 31% 2015-10-29T01:42:35.004Z I - [rsSync] Index Build: 210400/292160 72% 2015-10-29T01:42:50.395Z I INDEX [rsSync] build index done. scanned 292160 total records. 20 secs 2015-10-29T01:42:51.101Z I INDEX [rsSync] build index on: sledge.scheduled_messages properties: { v: 1, key: { executed_at: 1, scheduled_at: 1 }, name: "executed_at_1_scheduled_at_1", ns: "sledge.scheduled_messages", background: true } 2015-10-29T01:42:51.101Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.103Z I INDEX [rsSync] build index on: sledge.scheduled_messages properties: { v: 1, key: { scheduled_at: 1, executed_at: 1 }, name: "scheduled_at_1_executed_at_1", ns: "sledge.scheduled_messages", background: true } 2015-10-29T01:42:51.103Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.109Z I INDEX [rsSync] build index done. scanned 0 total records. 0 secs 2015-10-29T01:42:51.112Z I INDEX [rsSync] build index on: sledge.sophos_articles properties: { v: 1, unique: true, key: { title: 1, description: 1, link: 1, pubDate: 1 }, name: "title_1_description_1_link_1_pubDate_1", ns: "sledge.sophos_articles", background: true } 2015-10-29T01:42:51.112Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.118Z I INDEX [rsSync] build index done. scanned 288 total records. 0 secs 2015-10-29T01:42:51.120Z I INDEX [rsSync] build index on: sledge.sophos_labs_threat_level properties: { v: 1, unique: true, key: { timestamp: 1 }, name: "timestamp_1", ns: "sledge.sophos_labs_threat_level", background: true } 2015-10-29T01:42:51.120Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.125Z I INDEX [rsSync] build index done. scanned 1 total records. 0 secs 2015-10-29T01:42:51.127Z I INDEX [rsSync] build index on: sledge.sophos_labs_threat_volumes properties: { v: 1, unique: true, key: { measured_at: 1 }, name: "measured_at_1", ns: "sledge.sophos_labs_threat_volumes", background: true } 2015-10-29T01:42:51.127Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.152Z I INDEX [rsSync] build index done. scanned 9894 total records. 0 secs 2015-10-29T01:42:51.153Z I REPL [rsSync] initial sync cloning indexes for : test 2015-10-29T01:42:51.204Z I INDEX [rsSync] build index on: test.customer_flags properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "test.customer_flags", sparse: false, background: true } 2015-10-29T01:42:51.205Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.208Z I INDEX [rsSync] build index done. scanned 54 total records. 0 secs 2015-10-29T01:42:51.211Z I INDEX [rsSync] build index on: test.customers properties: { v: 1, key: { licenses.expires_on: 1 }, name: "licenses.expires_on_1", ns: "test.customers", sparse: false, background: true } 2015-10-29T01:42:51.211Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.213Z I INDEX [rsSync] build index on: test.customers properties: { v: 1, key: { beta: 1 }, name: "beta_1", ns: "test.customers", sparse: true, background: true } 2015-10-29T01:42:51.213Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.219Z I INDEX [rsSync] build index done. scanned 3 total records. 0 secs 2015-10-29T01:42:51.222Z I INDEX [rsSync] build index on: test.deployment_tokens properties: { v: 1, unique: true, key: { registration_token: 1 }, name: "registration_token_1", ns: "test.deployment_tokens", sparse: false, background: true } 2015-10-29T01:42:51.222Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.225Z I INDEX [rsSync] build index on: test.deployment_tokens properties: { v: 1, key: { user_id: 1 }, name: "user_id_1", ns: "test.deployment_tokens", sparse: false, background: true } 2015-10-29T01:42:51.225Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.227Z I INDEX [rsSync] build index on: test.deployment_tokens properties: { v: 1, key: { customer_id: 1, user_id: 1, beta: 1 }, name: "customer_id_1_user_id_1_beta_1", ns: "test.deployment_tokens", sparse: false, background: true } 2015-10-29T01:42:51.227Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.236Z I INDEX [rsSync] build index done. scanned 3 total records. 0 secs 2015-10-29T01:42:51.239Z I INDEX [rsSync] build index on: test.directory_objects properties: { v: 1, key: { c: 1, _type: 1 }, name: "c_1__type_1", ns: "test.directory_objects", sparse: false, background: true } 2015-10-29T01:42:51.239Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.241Z I INDEX [rsSync] build index on: test.directory_objects properties: { v: 1, key: { c: 1, _type: 1, name_lowercase: 1 }, name: "c_1__type_1_name_lowercase_1", ns: "test.directory_objects", sparse: false, background: true } 2015-10-29T01:42:51.241Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.243Z I INDEX [rsSync] build index on: test.directory_objects properties: { v: 1, key: { c: 1, _type: 1, search_tokens: 1 }, name: "c_1__type_1_search_tokens_1", ns: "test.directory_objects", sparse: false, background: true } 2015-10-29T01:42:51.243Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.246Z I INDEX [rsSync] build index on: test.directory_objects properties: { v: 1, key: { c: 1, _type: 1, external_id: 1 }, name: "c_1__type_1_external_id_1", ns: "test.directory_objects", sparse: false, background: true } 2015-10-29T01:42:51.246Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.248Z I INDEX [rsSync] build index on: test.directory_objects properties: { v: 1, key: { c: 1, _type: 1, origin: 1 }, name: "c_1__type_1_origin_1", ns: "test.directory_objects", sparse: false, background: true } 2015-10-29T01:42:51.248Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.250Z I INDEX [rsSync] build index on: test.directory_objects properties: { v: 1, key: { c: 1, _type: 1, email: 1 }, name: "c_1__type_1_email_1", ns: "test.directory_objects", sparse: false, background: true } 2015-10-29T01:42:51.250Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.252Z I INDEX [rsSync] build index on: test.directory_objects properties: { v: 1, key: { c: 1, _type: 1, dn: 1 }, name: "c_1__type_1_dn_1", ns: "test.directory_objects", sparse: false, background: true } 2015-10-29T01:42:51.252Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.312Z I INDEX [rsSync] build index done. scanned 7 total records. 0 secs 2015-10-29T01:42:51.316Z I INDEX [rsSync] build index on: test.directory_user_matches properties: { v: 1, key: { local_endpoint_id: 1 }, name: "local_endpoint_id_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-29T01:42:51.316Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.318Z I INDEX [rsSync] build index on: test.directory_user_matches properties: { v: 1, key: { name_lowercase: 1 }, name: "name_lowercase_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-29T01:42:51.318Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.321Z I INDEX [rsSync] build index on: test.directory_user_matches properties: { v: 1, unique: true, key: { name_lowercase: 1, c: 1 }, name: "name_lowercase_1_c_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-29T01:42:51.321Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.323Z I INDEX [rsSync] build index on: test.directory_user_matches properties: { v: 1, key: { user_id: 1 }, name: "user_id_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-29T01:42:51.323Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.325Z I INDEX [rsSync] build index on: test.directory_user_matches properties: { v: 1, key: { customer_id: 1, name_lowercase: 1 }, name: "customer_id_1_name_lowercase_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-29T01:42:51.325Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.327Z I INDEX [rsSync] build index on: test.directory_user_matches properties: { v: 1, key: { c: 1, local_endpoint_id: 1 }, name: "c_1_local_endpoint_id_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-29T01:42:51.327Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.329Z I INDEX [rsSync] build index on: test.directory_user_matches properties: { v: 1, key: { c: 1, name_lowercase: 1 }, name: "c_1_name_lowercase_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-29T01:42:51.329Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.332Z I INDEX [rsSync] build index on: test.directory_user_matches properties: { v: 1, key: { c: 1, external_id: 1 }, name: "c_1_external_id_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-29T01:42:51.332Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.334Z I INDEX [rsSync] build index on: test.directory_user_matches properties: { v: 1, key: { c: 1, origin: 1 }, name: "c_1_origin_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-29T01:42:51.334Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.336Z I INDEX [rsSync] build index on: test.directory_user_matches properties: { v: 1, key: { c: 1, name: 1 }, name: "c_1_name_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-29T01:42:51.336Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.338Z I INDEX [rsSync] build index on: test.directory_user_matches properties: { v: 1, key: { c: 1, dn: 1 }, name: "c_1_dn_1", ns: "test.directory_user_matches", sparse: false, background: true } 2015-10-29T01:42:51.338Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.375Z I INDEX [rsSync] build index done. scanned 3 total records. 0 secs 2015-10-29T01:42:51.378Z I INDEX [rsSync] build index on: test.endpoints properties: { v: 1, key: { user_match_id: 1 }, name: "user_match_id_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-29T01:42:51.378Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.380Z I INDEX [rsSync] build index on: test.endpoints properties: { v: 1, key: { domain_name: 1, computer_name: 1, operating_system: 1 }, name: "domain_name_1_computer_name_1_operating_system_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-29T01:42:51.380Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.382Z I INDEX [rsSync] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-29T01:42:51.382Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.384Z I INDEX [rsSync] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, feature_codes: 1 }, name: "customer_id_1_feature_codes_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-29T01:42:51.384Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.387Z I INDEX [rsSync] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, label: 1 }, name: "customer_id_1_label_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-29T01:42:51.387Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.389Z I INDEX [rsSync] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, label_lowercase: 1 }, name: "customer_id_1_label_lowercase_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-29T01:42:51.389Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.391Z I INDEX [rsSync] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, user_match_id: 1 }, name: "customer_id_1_user_match_id_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-29T01:42:51.391Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.393Z I INDEX [rsSync] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, search_tokens: 1 }, name: "customer_id_1_search_tokens_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-29T01:42:51.393Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.396Z I INDEX [rsSync] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, transport: 1 }, name: "customer_id_1_transport_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-29T01:42:51.396Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.398Z I INDEX [rsSync] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, endpoint_type: 1 }, name: "customer_id_1_endpoint_type_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-29T01:42:51.398Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.400Z I INDEX [rsSync] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, info.platform: 1 }, name: "customer_id_1_info.platform_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-29T01:42:51.400Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.402Z I INDEX [rsSync] build index on: test.endpoints properties: { v: 1, key: { customer_id: 1, registered_at: 1 }, name: "customer_id_1_registered_at_1", ns: "test.endpoints", sparse: false, background: true } 2015-10-29T01:42:51.402Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.404Z I INDEX [rsSync] build index on: test.endpoints properties: { v: 1, key: { last_connected_from: 1 }, name: "last_connected_from_1", ns: "test.endpoints", sparse: true, background: true } 2015-10-29T01:42:51.404Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.407Z I INDEX [rsSync] build index on: test.endpoints properties: { v: 1, key: { beta: 1 }, name: "beta_1", ns: "test.endpoints", sparse: true, background: true } 2015-10-29T01:42:51.407Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.459Z I INDEX [rsSync] build index done. scanned 3 total records. 0 secs 2015-10-29T01:42:51.462Z I INDEX [rsSync] build index on: test.event_bases properties: { v: 1, key: { customer_id: 1, severity: 1, reported_at: 1, _type: 1 }, name: "customer_id_1_severity_1_reported_at_1__type_1", ns: "test.event_bases", sparse: false, background: true } 2015-10-29T01:42:51.462Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.464Z I INDEX [rsSync] build index on: test.event_bases properties: { v: 1, key: { customer_id: 1, reported_at: -1, _type: 1 }, name: "customer_id_1_reported_at_-1__type_1", ns: "test.event_bases", sparse: false, background: true } 2015-10-29T01:42:51.464Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.467Z I INDEX [rsSync] build index on: test.event_bases properties: { v: 1, key: { _type: 1, reported_at: 1 }, name: "_type_1_reported_at_1", ns: "test.event_bases", sparse: false, background: true } 2015-10-29T01:42:51.467Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.469Z I INDEX [rsSync] build index on: test.event_bases properties: { v: 1, key: { reported_at: 1 }, name: "reported_at_1", ns: "test.event_bases", sparse: false, background: true } 2015-10-29T01:42:51.469Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.473Z I INDEX [rsSync] build index on: test.event_bases properties: { v: 1, key: { customer_id: 1, inserted_at: 1 }, name: "customer_id_1_inserted_at_1", ns: "test.event_bases", sparse: true, background: true } 2015-10-29T01:42:51.473Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.475Z I INDEX [rsSync] build index on: test.event_bases properties: { v: 1, key: { endpoint_id: 1, _type: 1, reported_at: 1 }, name: "endpoint_id_1__type_1_reported_at_1", ns: "test.event_bases", sparse: false, background: true } 2015-10-29T01:42:51.475Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.477Z I INDEX [rsSync] build index on: test.event_bases properties: { v: 1, key: { user_match_id: 1 }, name: "user_match_id_1", ns: "test.event_bases", sparse: true, background: true } 2015-10-29T01:42:51.477Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.502Z I INDEX [rsSync] build index done. scanned 3 total records. 0 secs 2015-10-29T01:42:51.505Z I INDEX [rsSync] build index on: test.global_exclusions properties: { v: 1, key: { customer_id: 1 }, name: "customer_id_1", ns: "test.global_exclusions", sparse: false, background: true } 2015-10-29T01:42:51.505Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.509Z I INDEX [rsSync] build index done. scanned 3 total records. 0 secs 2015-10-29T01:42:51.512Z I INDEX [rsSync] build index on: test.mcs_policy_bases properties: { v: 1, key: { customer_id: 1, _type: 1 }, name: "customer_id_1__type_1", ns: "test.mcs_policy_bases", sparse: false, background: true } 2015-10-29T01:42:51.512Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.515Z I INDEX [rsSync] build index done. scanned 12 total records. 0 secs 2015-10-29T01:42:51.518Z I INDEX [rsSync] build index on: test.policies properties: { v: 1, key: { customer_id: 1, target: 1, priority: -1 }, name: "customer_id_1_target_1_priority_-1", ns: "test.policies", sparse: false, background: true } 2015-10-29T01:42:51.518Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.521Z I INDEX [rsSync] build index on: test.policies properties: { v: 1, key: { enabled: 1, disable_at: 1 }, name: "enabled_1_disable_at_1", ns: "test.policies", sparse: false, background: true } 2015-10-29T01:42:51.521Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.529Z I INDEX [rsSync] build index done. scanned 24 total records. 0 secs 2015-10-29T01:42:51.532Z I INDEX [rsSync] build index on: test.policy_assignments properties: { v: 1, key: { endpoint_id: 1, policy_type: 1 }, name: "endpoint_id_1_policy_type_1", ns: "test.policy_assignments", sparse: false, background: true } 2015-10-29T01:42:51.532Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.534Z I INDEX [rsSync] build index on: test.policy_assignments properties: { v: 1, key: { endpoint_id: 1 }, name: "endpoint_id_1", ns: "test.policy_assignments", sparse: false, background: true } 2015-10-29T01:42:51.534Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.542Z I INDEX [rsSync] build index done. scanned 30 total records. 0 secs 2015-10-29T01:42:51.545Z I INDEX [rsSync] build index on: test.rendered_policies properties: { v: 1, key: { policy_fragment_ids: 1 }, name: "policy_fragment_ids_1", ns: "test.rendered_policies", sparse: false, background: true } 2015-10-29T01:42:51.545Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.549Z I INDEX [rsSync] build index done. scanned 23 total records. 0 secs 2015-10-29T01:42:51.549Z I REPL [rsSync] initial sync cloning indexes for : upe_dev 2015-10-29T01:42:51.581Z I INDEX [rsSync] build index on: upe_dev.customers properties: { v: 1, key: { licenses.expires_on: 1 }, name: "licenses.expires_on_1", ns: "upe_dev.customers", sparse: false, background: true } 2015-10-29T01:42:51.581Z I INDEX [rsSync] building index using bulk method 2015-10-29T01:42:51.585Z I INDEX [rsSync] build index done. scanned 2 total records. 0 secs 2015-10-29T01:42:51.585Z I REPL [rsSync] initial sync cloning indexes for : zero_dev 2015-10-29T01:42:51.618Z I REPL [rsSync] oplog sync 3 of 3 2015-10-29T01:42:55.708Z I REPL [rsSync] initial sync finishing up 2015-10-29T01:42:55.709Z I REPL [rsSync] replSet set minValid=563179d8:131 2015-10-29T01:42:55.710Z I REPL [rsSync] initial sync done 2015-10-29T01:42:55.719Z I REPL [ReplicationExecutor] transition to RECOVERING 2015-10-29T01:42:55.720Z I REPL [ReplicationExecutor] could not find member to sync from 2015-10-29T01:42:55.720Z I REPL [ReplicationExecutor] transition to SECONDARY 2015-10-29T01:42:57.720Z I REPL [ReplicationExecutor] syncing from: mongodb-upe_dccb-2.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 2015-10-29T01:42:58.126Z I REPL [SyncSourceFeedback] replset setting syncSourceFeedback to mongodb-upe_dccb-2.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 2015-10-29T11:22:09.602Z I QUERY [conn3878] assertion 13435 not master and slaveOk=false ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-29T13:09:56.476Z I REPL [conn4423] Stopping replication applier threads 2015-10-29T13:09:56.571Z I CONTROL [conn4423] now exiting 2015-10-29T13:09:56.571Z I NETWORK [conn4423] shutdown: going to close listening sockets... 2015-10-29T13:09:56.571Z I NETWORK [conn4423] closing listening socket: 6 2015-10-29T13:09:56.571Z I NETWORK [conn4423] closing listening socket: 7 2015-10-29T13:09:56.571Z I NETWORK [conn4423] removing socket file: /tmp/mongodb-27017.sock 2015-10-29T13:09:56.571Z I NETWORK [conn4423] shutdown: going to flush diaglog... 2015-10-29T13:09:56.571Z I NETWORK [conn4423] shutdown: going to close sockets... 2015-10-29T13:09:59.600Z I CONTROL [conn4423] dbexit: rc: 0 2015-10-29T13:16:46.435Z I CONTROL ***** SERVER RESTARTED ***** 2015-10-29T13:16:46.578Z I CONTROL [initandlisten] MongoDB starting : pid=72282 port=27017 dbpath=/mongodata/db 64-bit host=ip-172-19-100-23 2015-10-29T13:16:46.578Z I CONTROL [initandlisten] db version v3.0.5 2015-10-29T13:16:46.578Z I CONTROL [initandlisten] git version: 8bc4ae20708dbb493cb09338d9e7be6698e4a3a3 2015-10-29T13:16:46.578Z I CONTROL [initandlisten] build info: Linux ip-10-230-137-3 3.4.43-43.43.amzn1.x86_64 #1 SMP Mon May 6 18:04:41 UTC 2013 x86_64 BOOST_LIB_VERSION=1_49 2015-10-29T13:16:46.578Z I CONTROL [initandlisten] allocator: tcmalloc 2015-10-29T13:16:46.578Z I CONTROL [initandlisten] options: { config: "/etc/mongod.conf", net: { port: 27017, ssl: { PEMKeyFile: "/mongodata/config/mongodb.pem", PEMKeyPassword: "", mode: "preferSSL" } }, operationProfiling: { mode: "off", slowOpThresholdMs: 100 }, processManagement: { fork: true, pidFilePath: "/var/run/mongodb/mongod.pid" }, replication: { oplogSizeMB: 20480, replSetName: "upe_dccb" }, security: { clusterAuthMode: "keyFile", keyFile: "/mongodata/config/keyfile" }, storage: { dbPath: "/mongodata/db", directoryPerDB: true, engine: "wiredTiger", wiredTiger: { collectionConfig: { blockCompressor: "snappy" }, engineConfig: { directoryForIndexes: true, journalCompressor: "snappy", statisticsLogDelaySecs: 0 }, indexConfig: { prefixCompression: true } } }, systemLog: { destination: "file", logAppend: true, logRotate: "reopen", path: "/data/log/mongodb/mongodb.log", timeStampFormat: "iso8601-utc", traceAllExceptions: false, verbosity: 0 } } 2015-10-29T13:16:46.595Z I NETWORK [initandlisten] waiting for connections on port 27017 ssl 2015-10-29T13:16:46.714Z I REPL [ReplicationExecutor] New replica set config in use: { _id: "upe_dccb", version: 11496652, members: [ { _id: 4, host: "mongodb-upe_dccb-1.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017", arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 5, host: "mongodb-upe_dccb-2.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017", arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 6, host: "mongodb-upe_dccb-3.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017", arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 } ], settings: { chainingAllowed: true, heartbeatTimeoutSecs: 10, getLastErrorModes: {}, getLastErrorDefaults: { w: 1, wtimeout: 0 } } } 2015-10-29T13:16:46.714Z I REPL [ReplicationExecutor] This node is mongodb-upe_dccb-1.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 in the config 2015-10-29T13:16:46.714Z I REPL [ReplicationExecutor] transition to STARTUP2 2015-10-29T13:16:46.714Z I REPL [ReplicationExecutor] Starting replication applier threads 2015-10-29T13:16:46.714Z I REPL [ReplicationExecutor] transition to RECOVERING 2015-10-29T13:16:46.743Z I REPL [ReplicationExecutor] Member mongodb-upe_dccb-2.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 is now in state SECONDARY 2015-10-29T13:16:46.747Z I REPL [ReplicationExecutor] Member mongodb-upe_dccb-3.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 is now in state PRIMARY 2015-10-29T13:16:49.715Z I REPL [ReplicationExecutor] syncing from: mongodb-upe_dccb-2.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 2015-10-29T13:16:50.319Z W REPL [rsBackgroundSync] we are too stale to use mongodb-upe_dccb-2.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 as a sync source 2015-10-29T13:16:50.320Z I REPL [ReplicationExecutor] syncing from: mongodb-upe_dccb-3.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 2015-10-29T13:16:50.858Z I REPL [ReplicationExecutor] could not find member to sync from 2015-10-29T13:16:50.858Z I REPL [rsBackgroundSync] replSet error RS102 too stale to catch up 2015-10-29T13:16:50.858Z I REPL [rsBackgroundSync] replSet our last optime : Oct 29 01:34:08 56317790:12b 2015-10-29T13:16:50.858Z I REPL [rsBackgroundSync] replSet oldest available is Oct 29 08:21:59 5631d727:5f 2015-10-29T13:16:50.858Z I REPL [rsBackgroundSync] replSet See http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember 2015-10-29T13:18:54.609Z I - [conn521] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T13:18:54.617Z I CONTROL [conn521] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FA1F4850000","o":"7DF5"},{"b":"7FA1F3214000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFEC412B000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FA1F4850000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FA1F45E3000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FA1F41FE000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FA1F3FF6000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FA1F3DF2000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FA1F3AEE000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FA1F37EC000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FA1F35D6000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FA1F3214000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FA1F4A6C000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FA1F2FC8000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FA1F2CE5000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FA1F2AE2000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FA1F28B0000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FA1F269A000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FA1F248B000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FA1F2288000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FA1F206E000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FA1F1E4D000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fa1f4857df5] libc.so.6(clone+0x6D) [0x7fa1f330abfd] ----- END BACKTRACE ----- 2015-10-29T13:18:54.617Z I QUERY [conn521] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-29T13:22:37.253Z I - [conn542] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T13:22:37.260Z I CONTROL [conn542] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FA1F4850000","o":"7DF5"},{"b":"7FA1F3214000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFEC412B000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FA1F4850000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FA1F45E3000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FA1F41FE000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FA1F3FF6000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FA1F3DF2000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FA1F3AEE000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FA1F37EC000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FA1F35D6000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FA1F3214000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FA1F4A6C000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FA1F2FC8000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FA1F2CE5000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FA1F2AE2000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FA1F28B0000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FA1F269A000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FA1F248B000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FA1F2288000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FA1F206E000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FA1F1E4D000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fa1f4857df5] libc.so.6(clone+0x6D) [0x7fa1f330abfd] ----- END BACKTRACE ----- 2015-10-29T13:22:37.261Z I QUERY [conn542] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-29T13:22:46.157Z I QUERY [conn542] assertion 13435 not master and slaveOk=false ns:sledge.mcs_commands query:{} 2015-10-29T13:22:54.284Z I QUERY [conn542] assertion 13436 not master or secondary; cannot currently read from this replSet member ns:sledge.mcs_commands query:{} 2015-10-29T13:26:50.952Z I REPL [ReplicationExecutor] syncing from: mongodb-upe_dccb-3.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 2015-10-29T13:26:51.343Z W REPL [rsBackgroundSync] we are too stale to use mongodb-upe_dccb-3.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 as a sync source 2015-10-29T13:26:51.343Z I REPL [ReplicationExecutor] syncing from: mongodb-upe_dccb-2.CloudStation.eu-west-1.dev3.hydra.sophos.com:27017 2015-10-29T13:26:52.074Z I REPL [ReplicationExecutor] could not find member to sync from 2015-10-29T13:26:52.074Z I REPL [rsBackgroundSync] replSet error RS102 too stale to catch up 2015-10-29T13:26:52.074Z I REPL [rsBackgroundSync] replSet our last optime : Oct 29 01:34:08 56317790:12b 2015-10-29T13:26:52.074Z I REPL [rsBackgroundSync] replSet oldest available is Oct 29 08:33:46 5631d9ea:14b 2015-10-29T13:26:52.074Z I REPL [rsBackgroundSync] replSet See http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember 2015-10-29T13:29:47.741Z I REPL [conn582] Stopping replication applier threads 2015-10-29T13:29:48.102Z I CONTROL [conn582] now exiting 2015-10-29T13:29:48.102Z I NETWORK [conn582] shutdown: going to close listening sockets... 2015-10-29T13:29:48.102Z I NETWORK [conn582] closing listening socket: 6 2015-10-29T13:29:48.103Z I NETWORK [conn582] closing listening socket: 7 2015-10-29T13:29:48.103Z I NETWORK [conn582] removing socket file: /tmp/mongodb-27017.sock 2015-10-29T13:29:48.103Z I NETWORK [conn582] shutdown: going to flush diaglog... 2015-10-29T13:29:48.103Z I NETWORK [conn582] shutdown: going to close sockets... 2015-10-29T13:29:48.129Z I CONTROL [conn582] dbexit: rc: 0 2015-10-29T13:30:54.093Z I CONTROL ***** SERVER RESTARTED ***** 2015-10-29T13:30:54.203Z I CONTROL [initandlisten] MongoDB starting : pid=73418 port=27017 dbpath=/mongodata/db 64-bit host=ip-172-19-100-23 2015-10-29T13:30:54.203Z I CONTROL [initandlisten] db version v3.0.5 2015-10-29T13:30:54.203Z I CONTROL [initandlisten] git version: 8bc4ae20708dbb493cb09338d9e7be6698e4a3a3 2015-10-29T13:30:54.203Z I CONTROL [initandlisten] build info: Linux ip-10-230-137-3 3.4.43-43.43.amzn1.x86_64 #1 SMP Mon May 6 18:04:41 UTC 2013 x86_64 BOOST_LIB_VERSION=1_49 2015-10-29T13:30:54.203Z I CONTROL [initandlisten] allocator: tcmalloc 2015-10-29T13:30:54.203Z I CONTROL [initandlisten] options: { config: "/etc/mongod.conf", net: { port: 27017, ssl: { PEMKeyFile: "/mongodata/config/mongodb.pem", PEMKeyPassword: "", mode: "preferSSL" } }, operationProfiling: { mode: "off", slowOpThresholdMs: 100 }, processManagement: { fork: true, pidFilePath: "/var/run/mongodb/mongod.pid" }, security: { clusterAuthMode: "keyFile", keyFile: "/mongodata/config/keyfile" }, storage: { dbPath: "/mongodata/db", directoryPerDB: true, engine: "wiredTiger", wiredTiger: { collectionConfig: { blockCompressor: "snappy" }, engineConfig: { directoryForIndexes: true, journalCompressor: "snappy", statisticsLogDelaySecs: 0 }, indexConfig: { prefixCompression: true } } }, systemLog: { destination: "file", logAppend: true, logRotate: "reopen", path: "/data/log/mongodb/mongodb.log", timeStampFormat: "iso8601-utc", traceAllExceptions: false, verbosity: 0 } } 2015-10-29T13:30:54.217Z I NETWORK [initandlisten] waiting for connections on port 27017 ssl 2015-10-29T13:31:42.818Z I - [conn92] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T13:31:42.825Z I CONTROL [conn92] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FC6CEE2E000","o":"7DF5"},{"b":"7FC6CD7F2000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFD3B360000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FC6CEE2E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FC6CEBC1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FC6CE7DC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FC6CE5D4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FC6CE3D0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FC6CE0CC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FC6CDDCA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FC6CDBB4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FC6CD7F2000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FC6CF04A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FC6CD5A6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FC6CD2C3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FC6CD0C0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FC6CCE8E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FC6CCC78000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FC6CCA69000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FC6CC866000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FC6CC64C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FC6CC42B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fc6cee35df5] libc.so.6(clone+0x6D) [0x7fc6cd8e8bfd] ----- END BACKTRACE ----- 2015-10-29T13:31:42.825Z I QUERY [conn92] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-29T14:34:09.512Z I - [conn21213] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T14:34:09.519Z I CONTROL [conn21213] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FC6CEE2E000","o":"7DF5"},{"b":"7FC6CD7F2000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFD3B360000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FC6CEE2E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FC6CEBC1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FC6CE7DC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FC6CE5D4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FC6CE3D0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FC6CE0CC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FC6CDDCA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FC6CDBB4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FC6CD7F2000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FC6CF04A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FC6CD5A6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FC6CD2C3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FC6CD0C0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FC6CCE8E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FC6CCC78000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FC6CCA69000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FC6CC866000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FC6CC64C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FC6CC42B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fc6cee35df5] libc.so.6(clone+0x6D) [0x7fc6cd8e8bfd] ----- END BACKTRACE ----- 2015-10-29T14:34:09.519Z I QUERY [conn21213] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B) } 2015-10-29T14:36:50.787Z I - [conn21213] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T14:36:50.796Z I CONTROL [conn21213] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"57DD05"},{"b":"400000","o":"578377"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FC6CEE2E000","o":"7DF5"},{"b":"7FC6CD7F2000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFD3B360000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FC6CEE2E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FC6CEBC1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FC6CE7DC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FC6CE5D4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FC6CE3D0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FC6CE0CC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FC6CDDCA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FC6CDBB4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FC6CD7F2000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FC6CF04A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FC6CD5A6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FC6CD2C3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FC6CD0C0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FC6CCE8E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FC6CCC78000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FC6CCA69000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FC6CC866000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FC6CC64C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FC6CC42B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZNK5mongo7FindCmd7explainEPNS_16OperationContextERKSsRKNS_7BSONObjENS_13ExplainCommon9VerbosityEPNS_14BSONObjBuilderE+0x345) [0x97dd05] mongod(_ZN5mongo10CmdExplain3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x177) [0x978377] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fc6cee35df5] libc.so.6(clone+0x6D) [0x7fc6cd8e8bfd] ----- END BACKTRACE ----- 2015-10-29T14:43:27.698Z I - [conn21213] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T14:43:27.706Z I CONTROL [conn21213] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FC6CEE2E000","o":"7DF5"},{"b":"7FC6CD7F2000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFD3B360000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FC6CEE2E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FC6CEBC1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FC6CE7DC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FC6CE5D4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FC6CE3D0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FC6CE0CC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FC6CDDCA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FC6CDBB4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FC6CD7F2000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FC6CF04A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FC6CD5A6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FC6CD2C3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FC6CD0C0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FC6CCE8E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FC6CCC78000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FC6CCA69000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FC6CC866000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FC6CC64C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FC6CC42B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fc6cee35df5] libc.so.6(clone+0x6D) [0x7fc6cd8e8bfd] ----- END BACKTRACE ----- 2015-10-29T14:43:27.706Z I QUERY [conn21213] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B), posted_app_id: "APPSPROXY" } 2015-10-29T14:43:47.098Z I - [conn21213] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T14:43:47.107Z I CONTROL [conn21213] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"57DD05"},{"b":"400000","o":"578377"},{"b":"400000","o":"5F1F04"},{"b":"400000","o":"5F2E8D"},{"b":"400000","o":"5F3B9B"},{"b":"400000","o":"7B884A"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FC6CEE2E000","o":"7DF5"},{"b":"7FC6CD7F2000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFD3B360000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FC6CEE2E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FC6CEBC1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FC6CE7DC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FC6CE5D4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FC6CE3D0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FC6CE0CC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FC6CDDCA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FC6CDBB4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FC6CD7F2000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FC6CF04A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FC6CD5A6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FC6CD2C3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FC6CD0C0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FC6CCE8E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FC6CCC78000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FC6CCA69000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FC6CC866000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FC6CC64C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FC6CC42B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZNK5mongo7FindCmd7explainEPNS_16OperationContextERKSsRKNS_7BSONObjENS_13ExplainCommon9VerbosityEPNS_14BSONObjBuilderE+0x345) [0x97dd05] mongod(_ZN5mongo10CmdExplain3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x177) [0x978377] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f1f04] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f2e8d] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f3b9b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbb884a] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fc6cee35df5] libc.so.6(clone+0x6D) [0x7fc6cd8e8bfd] ----- END BACKTRACE ----- 2015-10-29T14:45:56.236Z I - [conn21213] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T14:45:56.244Z I CONTROL [conn21213] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FC6CEE2E000","o":"7DF5"},{"b":"7FC6CD7F2000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFD3B360000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FC6CEE2E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FC6CEBC1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FC6CE7DC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FC6CE5D4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FC6CE3D0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FC6CE0CC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FC6CDDCA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FC6CDBB4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FC6CD7F2000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FC6CF04A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FC6CD5A6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FC6CD2C3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FC6CD0C0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FC6CCE8E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FC6CCC78000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FC6CCA69000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FC6CC866000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FC6CC64C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FC6CC42B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fc6cee35df5] libc.so.6(clone+0x6D) [0x7fc6cd8e8bfd] ----- END BACKTRACE ----- 2015-10-29T14:45:56.244Z I QUERY [conn21213] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B), posted_app_id: "APPSPROXY" } 2015-10-29T14:46:11.724Z I - [conn21213] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T14:46:11.731Z I CONTROL [conn21213] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FC6CEE2E000","o":"7DF5"},{"b":"7FC6CD7F2000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFD3B360000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FC6CEE2E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FC6CEBC1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FC6CE7DC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FC6CE5D4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FC6CE3D0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FC6CE0CC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FC6CDDCA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FC6CDBB4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FC6CD7F2000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FC6CF04A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FC6CD5A6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FC6CD2C3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FC6CD0C0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FC6CCE8E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FC6CCC78000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FC6CCA69000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FC6CC866000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FC6CC64C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FC6CC42B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fc6cee35df5] libc.so.6(clone+0x6D) [0x7fc6cd8e8bfd] ----- END BACKTRACE ----- 2015-10-29T14:46:11.731Z I QUERY [conn21213] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B), posted_app_id: "APPSPROXY", body_sha: "d822187be9f68fc7eb02b9cd0c8c4518cf91e8ac" } 2015-10-29T14:46:25.101Z I - [conn21213] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T14:46:25.109Z I CONTROL [conn21213] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FC6CEE2E000","o":"7DF5"},{"b":"7FC6CD7F2000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFD3B360000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FC6CEE2E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FC6CEBC1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FC6CE7DC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FC6CE5D4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FC6CE3D0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FC6CE0CC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FC6CDDCA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FC6CDBB4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FC6CD7F2000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FC6CF04A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FC6CD5A6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FC6CD2C3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FC6CD0C0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FC6CCE8E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FC6CCC78000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FC6CCA69000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FC6CC866000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FC6CC64C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FC6CC42B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fc6cee35df5] libc.so.6(clone+0x6D) [0x7fc6cd8e8bfd] ----- END BACKTRACE ----- 2015-10-29T14:46:25.109Z I QUERY [conn21213] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B), body_sha: "d822187be9f68fc7eb02b9cd0c8c4518cf91e8ac" } 2015-10-29T14:46:37.107Z I - [conn21213] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T14:46:37.115Z I CONTROL [conn21213] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FC6CEE2E000","o":"7DF5"},{"b":"7FC6CD7F2000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFD3B360000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FC6CEE2E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FC6CEBC1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FC6CE7DC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FC6CE5D4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FC6CE3D0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FC6CE0CC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FC6CDDCA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FC6CDBB4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FC6CD7F2000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FC6CF04A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FC6CD5A6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FC6CD2C3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FC6CD0C0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FC6CCE8E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FC6CCC78000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FC6CCA69000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FC6CC866000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FC6CC64C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FC6CC42B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fc6cee35df5] libc.so.6(clone+0x6D) [0x7fc6cd8e8bfd] ----- END BACKTRACE ----- 2015-10-29T14:46:37.115Z I QUERY [conn21213] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B), body_sha: "d822187be9f68fc7eb02b9cd0c8c4518cf91e8ad" } 2015-10-29T14:46:51.338Z I - [conn21213] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T14:46:51.346Z I CONTROL [conn21213] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FC6CEE2E000","o":"7DF5"},{"b":"7FC6CD7F2000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFD3B360000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FC6CEE2E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FC6CEBC1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FC6CE7DC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FC6CE5D4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FC6CE3D0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FC6CE0CC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FC6CDDCA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FC6CDBB4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FC6CD7F2000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FC6CF04A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FC6CD5A6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FC6CD2C3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FC6CD0C0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FC6CCE8E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FC6CCC78000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FC6CCA69000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FC6CC866000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FC6CC64C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FC6CC42B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fc6cee35df5] libc.so.6(clone+0x6D) [0x7fc6cd8e8bfd] ----- END BACKTRACE ----- 2015-10-29T14:46:51.346Z I QUERY [conn21213] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B), body_sha: "d822187be9f68fc7eb02b9cd0c8c4518cf91e8ac" } 2015-10-29T14:53:26.243Z I - [conn21213] Assertion: 28556:Didn't find RecordId in WiredTigerRecordStore 2015-10-29T14:53:26.250Z I CONTROL [conn21213] ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B77CA9"},{"b":"400000","o":"B16B61"},{"b":"400000","o":"AFBDBF"},{"b":"400000","o":"98FF0C"},{"b":"400000","o":"52CB70"},{"b":"400000","o":"61CB0B"},{"b":"400000","o":"62DCD4"},{"b":"400000","o":"62E5C7"},{"b":"400000","o":"7EAAE2"},{"b":"400000","o":"7EB14C"},{"b":"400000","o":"7EB47F"},{"b":"400000","o":"7BCCFF"},{"b":"400000","o":"7B8736"},{"b":"400000","o":"6CEB60"},{"b":"400000","o":"429DCD"},{"b":"400000","o":"B2AC4B"},{"b":"7FC6CEE2E000","o":"7DF5"},{"b":"7FC6CD7F2000","o":"F6BFD"}],"processInfo":{ "mongodbVersion" : "3.0.5", "gitVersion" : "8bc4ae20708dbb493cb09338d9e7be6698e4a3a3", "uname" : { "sysname" : "Linux", "release" : "4.1.7-15.23.amzn1.x86_64", "version" : "#1 SMP Mon Sep 14 23:20:33 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E8A046532FE2C5A8474250BA8E5B72C6C84B9D68" }, { "b" : "7FFD3B360000", "elfType" : 3, "buildId" : "AF79256455DAB4D8F13725CDA5E71975F69FED3F" }, { "b" : "7FC6CEE2E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "21054340E69FD5F7DDC9DECEF7951CF3F3D2E78E" }, { "b" : "7FC6CEBC1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FC6CE7DC000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FC6CE5D4000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "8A6EB7521ABE0518C2803504B52F18412D69C713" }, { "b" : "7FC6CE3D0000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "7030B651A4F5D6F32A8D83F3541990E342C50E6C" }, { "b" : "7FC6CE0CC000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7FC6CDDCA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "EB09196C8A859427FD964381CDAC80EC0C4C4296" }, { "b" : "7FC6CDBB4000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7FC6CD7F2000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "0EB7F7FA93C9C3E017F7257AE97415730345767B" }, { "b" : "7FC6CF04A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "F451C8F279B6860B00F835764FFBA0D54879E1C6" }, { "b" : "7FC6CD5A6000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FC6CD2C3000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FC6CD0C0000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FC6CCE8E000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FC6CCC78000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FC6CCA69000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FC6CC866000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FC6CC64C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F586C9A0950DFD60125B2F56B0EC6D690128B1FC" }, { "b" : "7FC6CC42B000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "C16D759FEE22609796EA66A648C297C1EB9191D1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77ca9] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b61] mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbdbf] mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd8ff0c] mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92cb70] mongod(_ZN5mongo10FetchStage4workEPm+0x2BB) [0xa1cb0b] mongod(_ZN5mongo14MultiPlanStage12workAllPlansEmPNS_15PlanYieldPolicyE+0xE4) [0xa2dcd4] mongod(_ZN5mongo14MultiPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0xC7) [0xa2e5c7] mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0x72) [0xbeaae2] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionERKSsNS0_11YieldPolicyEPPS0_+0x7C) [0xbeb14c] mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextEPNS_10WorkingSetEPNS_9PlanStageEPNS_13QuerySolutionEPNS_14CanonicalQueryEPKNS_10CollectionENS0_11YieldPolicyEPPS0_+0x7F) [0xbeb47f] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionEPNS_14CanonicalQueryENS_12PlanExecutor11YieldPolicyEPPS6_m+0xCF) [0xbbccff] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x666) [0xbb8736] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xaceb60] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829dcd] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf2ac4b] libpthread.so.0(+0x7DF5) [0x7fc6cee35df5] libc.so.6(clone+0x6D) [0x7fc6cd8e8bfd] ----- END BACKTRACE ----- 2015-10-29T14:53:26.250Z I QUERY [conn21213] assertion 28556 Didn't find RecordId in WiredTigerRecordStore ns:sledge.mcs_commands query:{ endpoint_id: BinData(3, 57A77FC68DB140A4BD97C7B6E0F98E9B), command_id: 7 }